def main():
parser = argparse.ArgumentParser(description="Launch rally-verify job.")
parser.add_argument("--mode", type=str, default="light",
help="Mode of job. The 'full' mode corresponds to the "
"full set of verifier tests. The 'light' mode "
"corresponds to the smoke set of verifier tests.",
choices=["light", "full"])
parser.add_argument("--compare", action="store_true",
help="Start the second verification to generate a "
"trends report for two verifications.")
# TODO(ylobankov): Remove hard-coded Tempest related things and make it
# configurable.
parser.add_argument("--ctx-create-resources", action="store_true",
help="Make Tempest context create needed resources "
"for the tests.")
args = parser.parse_args()
steps = run(args)
results = [step.to_html() for step in steps]
template = utils.get_template("ci/index_verify.html")
with open(os.path.join(Step.BASE_DIR, "extra/index.html"), "w") as f:
f.write(template.render(steps=results))
if len([None for step in steps
if step.result["status"] == Status.PASS]) == len(steps):
return 0
return 1
def plot(tasks_results, include_libs=False):
extended_results = _extend_results(tasks_results)
template = ui_utils.get_template("task/report.html")
source, data = _process_tasks(extended_results)
return template.render(source=json.dumps(source),
data=json.dumps(data),
include_libs=include_libs)
def plot(tasks_results, include_libs=False):
# NOTE(amaretskiy): Transform generic results into extended
# results, so they can be processed by charts classes
extended_results = []
for result in tasks_results:
generic = {
"id": None,
"task_uuid": None,
"key": result["key"],
"data": {
"sla": result["sla"],
"raw": result["result"],
"full_duration": result[
"full_duration"],
"load_duration": result[
"load_duration"]},
"created_at": None,
"updated_at": None}
extended_results.extend(
objects.Task.extend_results([generic]))
template = ui_utils.get_template("task/report.html")
source, data = _process_tasks(extended_results)
return template.render(source=json.dumps(source), data=json.dumps(data),
include_libs=include_libs)
def trends(tasks_results):
trends = Trends()
for i, scenario in enumerate(_extend_results(tasks_results), 1):
trends.add_result(scenario)
template = ui_utils.get_template("task/trends.html")
return template.render(version=version.version_string(),
data=json.dumps(trends.get_data()))
def plot(tasks_results, include_libs=False):
# NOTE(amaretskiy): Transform generic results into extended
# results, so they can be processed by charts classes
extended_results = []
for result in tasks_results:
generic = {
"id": None,
"task_uuid": None,
"key": result["key"],
"data": {
"sla": result["sla"],
"raw": result["result"],
"full_duration": result["full_duration"],
"load_duration": result["load_duration"]
},
"created_at": None,
"updated_at": None
}
extended_results.extend(objects.Task.extend_results([generic]))
template = ui_utils.get_template("task/report.html")
source, data = _process_tasks(extended_results)
return template.render(source=json.dumps(source),
data=json.dumps(data),
include_libs=include_libs)
def main():
parser = argparse.ArgumentParser(description="Launch rally-verify job.")
parser.add_argument("--mode",
type=str,
default="light",
help="Mode of job. The 'full' mode corresponds to the "
"full set of verifier tests. The 'light' mode "
"corresponds to the smoke set of verifier tests.",
choices=["light", "full"])
parser.add_argument("--compare",
action="store_true",
help="Start the second verification to generate a "
"trends report for two verifications.")
# TODO(ylobankov): Remove hard-coded Tempest related things and make it
# configurable.
parser.add_argument("--ctx-create-resources",
action="store_true",
help="Make Tempest context create needed resources "
"for the tests.")
args = parser.parse_args()
steps = run(args)
results = [step.to_html() for step in steps]
template = utils.get_template("ci/index_verify.html")
with open(os.path.join(Step.BASE_DIR, "extra/index.html"), "w") as f:
f.write(template.render(steps=results))
if len([None for step in steps
if step.result["status"] == Status.PASS]) == len(steps):
return 0
return 1
def trends(tasks_results):
trends = Trends()
for i, scenario in enumerate(_extend_results(tasks_results), 1):
trends.add_result(scenario)
template = ui_utils.get_template("task/trends.html")
return template.render(version=version.version_string(),
data=json.dumps(trends.get_data()))
def plot(tasks_results, include_libs=False):
extended_results = _extend_results(tasks_results)
template = ui_utils.get_template("task/report.html")
source, data = _process_tasks(extended_results)
return template.render(version=version.version_string(),
source=json.dumps(source),
data=json.dumps(data),
include_libs=include_libs)
def generate(self):
report = self._generate()
uuids = report["verifications"].keys()
show_comparison_note = False
for test in report["tests"].values():
# make as much as possible processing here to reduce processing
# at JS side
test["has_details"] = False
for test_info in test["by_verification"].values():
if "details" not in test_info:
test_info["details"] = None
elif not test["has_details"]:
test["has_details"] = True
durations = []
# iter by uuids to store right order for comparison
for uuid in uuids:
if uuid in test["by_verification"]:
durations.append(test["by_verification"][uuid]["duration"])
if float(durations[-1]) < 0.001:
durations[-1] = "0"
# not to display such little duration in the report
test["by_verification"][uuid]["duration"] = ""
if len(durations) > 1 and not (
durations[0] == "0" and durations[-1] == "0"):
# compare result with result of the first verification
diff = float(durations[-1]) - float(durations[0])
result = "%s (" % durations[-1]
if diff >= 0:
result += "+"
result += "%s)" % diff
test["by_verification"][uuid]["duration"] = result
if not show_comparison_note and len(durations) > 2:
# NOTE(andreykurilin): only in case of comparison of more
# than 2 results of the same test we should display a note
# about the comparison strategy
show_comparison_note = True
template = ui_utils.get_template("verification/report.html")
context = {"uuids": uuids,
"verifications": report["verifications"],
"tests": report["tests"],
"show_comparison_note": show_comparison_note}
raw_report = template.render(data=json.dumps(context),
include_libs=self.INCLUDE_LIBS)
# in future we will support html_static and will need to save more
# files
if self.output_destination:
return {"files": {self.output_destination: raw_report},
"open": self.output_destination}
else:
return {"print": raw_report}
def generate(self):
report = self._generate()
uuids = report["verifications"].keys()
show_comparison_note = False
for test in report["tests"].values():
# make as much as possible processing here to reduce processing
# at JS side
test["has_details"] = False
for test_info in test["by_verification"].values():
if "details" not in test_info:
test_info["details"] = None
elif not test["has_details"]:
test["has_details"] = True
durations = []
# iter by uuids to store right order for comparison
for uuid in uuids:
if uuid in test["by_verification"]:
durations.append(test["by_verification"][uuid]["duration"])
if float(durations[-1]) < 0.001:
durations[-1] = "0"
# not to display such little duration in the report
test["by_verification"][uuid]["duration"] = ""
if len(durations) > 1 and not (
durations[0] == "0" and durations[-1] == "0"):
# compare result with result of the first verification
diff = float(durations[-1]) - float(durations[0])
result = "%s (" % durations[-1]
if diff >= 0:
result += "+"
result += "%s)" % diff
test["by_verification"][uuid]["duration"] = result
if not show_comparison_note and len(durations) > 2:
# NOTE(andreykurilin): only in case of comparison of more
# than 2 results of the same test we should display a note
# about the comparison strategy
show_comparison_note = True
template = ui_utils.get_template("verification/report.html")
context = {"uuids": list(uuids),
"verifications": report["verifications"],
"tests": report["tests"],
"show_comparison_note": show_comparison_note}
raw_report = template.render(data=json.dumps(context),
include_libs=self.INCLUDE_LIBS)
# in future we will support html_static and will need to save more
# files
if self.output_destination:
return {"files": {self.output_destination: raw_report},
"open": self.output_destination}
else:
return {"print": raw_report}
def trends(tasks):
trends = Trends()
for task in tasks:
for workload in itertools.chain(
*[s["workloads"] for s in task["subtasks"]]):
trends.add_result(task["uuid"], workload)
template = ui_utils.get_template("task/trends.html")
return template.render(version=version.version_string(),
data=json.dumps(trends.get_data()))
def to_html(self):
"""Make HTML report."""
template = utils.get_template("verification/report.html")
context = {
"uuids": self._uuids,
"verifications": self._runs,
"tests": self._tests
}
return template.render(data=json.dumps(context), include_libs=False)
def trends(tasks):
trends = Trends()
for task in tasks:
for workload in itertools.chain(
*[s["workloads"] for s in task["subtasks"]]):
trends.add_result(workload)
template = ui_utils.get_template("task/trends.html")
return template.render(version=version.version_string(),
data=json.dumps(trends.get_data()))
def generate_report(results):
"""Generates HTML report from test results in JSON format."""
tests = []
for i, name in enumerate(sorted(results["test_cases"])):
test = results["test_cases"][name]
if "tags" in test:
name = "%(name)s [%(tags)s]" % {
"name": name,
"tags": ", ".join(test["tags"])
}
if "traceback" in test:
output = utils.escape(test["traceback"])
elif "reason" in test:
matcher = SKIP_RE.match(test["reason"])
if matcher:
href = LAUNCHPAD_BUG_LINK.format(matcher.group("bug_number"))
output = re.sub(matcher.group("bug_number"), href,
test["reason"])
else:
output = utils.escape(test["reason"])
else:
output = ""
tests.append({
"id": i,
"time": test["time"],
"name": name,
"output": output,
"status": test["status"]
})
template = ui_utils.get_template("verification/report.mako")
return template.render(
report={
"tests":
tests,
"total":
results["tests"],
"time":
"{0} ({1} s)".format(
datetime.timedelta(
seconds=round(float(results["time"]))), results["time"]),
"success":
results["success"],
"failures":
results["failures"],
"skipped":
results["skipped"],
"expected_failures":
results["expected_failures"],
"unexpected_success":
results["unexpected_success"]
})
def create_report(results):
template_kw = {
"heading": {
"title": __title__,
"description": __description__,
"parameters": [("Difference Count", len(results))]
},
"generator": "compare2html %s" % __version__,
"results": results
}
template = ui_utils.get_template("verification/compare.mako")
output = template.render(**template_kw)
return output.encode("utf8")
def main():
statuses = []
org, project = get_project_name()
base = os.environ.get("BASE")
if base:
base_jobs_dir = os.path.join(base, "new", project)
else:
base_jobs_dir = os.path.realpath(".")
rally_root = "/home/rally/rally/"
if not os.path.exists(rally_root):
rally_root = os.environ["BASE"] + "/new/rally/"
jobs_dir = os.path.join(base_jobs_dir, "rally-jobs")
if not os.path.exists(jobs_dir):
# fallback to legacy path
jobs_dir = os.path.join(base_jobs_dir, "rally-scenarios")
if not os.path.exists(jobs_dir):
raise Exception("Rally jobs directory does not exist.")
for directory in ("plugins", "extra"):
dst = os.path.expanduser("~/.rally/%s" % directory)
try:
shutil.copytree(os.path.join(jobs_dir, directory), dst)
except OSError as e:
if e.errno != errno.EEXIST:
raise
scenario = os.environ.get("RALLY_SCENARIO", project).rsplit(".", 1)
scenario_name = scenario.pop(0)
scenario_ext = scenario.pop() if scenario else "yaml"
print("Processing scenario %s" % scenario_name)
for fname in os.listdir(jobs_dir):
print("Processing %s" % fname)
if fname.startswith(scenario_name):
tags = fname[len(scenario_name):-len(scenario_ext) - 1].split("_")
statuses.append(run_task(os.path.join(jobs_dir, fname), tags))
else:
print("Ignoring file %s" % fname)
print("Exit statuses: %r" % statuses)
template = utils.get_template("ci/index.html")
with open("rally-plot/extra/index.html", "w") as output:
output.write(template.render())
return any(statuses)
def plot(tasks_results, include_libs=False):
source = _make_source(tasks_results)
tasks = []
subtasks = []
workloads = []
for task in tasks_results:
tasks.append(task)
for subtask in tasks[-1]["subtasks"]:
workloads.extend(subtask.pop("workloads"))
subtasks.extend(tasks[-1].pop("subtasks"))
template = ui_utils.get_template("task/report.html")
data = _process_workloads(workloads)
return template.render(version=version.version_string(),
source=json.dumps(source),
data=json.dumps(data),
include_libs=include_libs)
def main():
statuses = []
org, project = get_project_name()
base = os.environ.get("BASE")
if base:
base_jobs_dir = os.path.join(base, "new", project)
else:
base_jobs_dir = os.path.realpath(".")
rally_root = "/home/rally/rally/"
if not os.path.exists(rally_root):
rally_root = os.environ["BASE"] + "/new/rally/"
jobs_dir = os.path.join(base_jobs_dir, "rally-jobs")
if not os.path.exists(jobs_dir):
# fallback to legacy path
jobs_dir = os.path.join(base_jobs_dir, "rally-scenarios")
if not os.path.exists(jobs_dir):
raise Exception("Rally jobs directory does not exist.")
for directory in ("plugins", "extra"):
dst = os.path.expanduser("~/.rally/%s" % directory)
try:
shutil.copytree(os.path.join(jobs_dir, directory), dst)
except OSError as e:
if e.errno != errno.EEXIST:
raise
scenario = os.environ.get("RALLY_SCENARIO", project).rsplit(".", 1)
scenario_name = scenario.pop(0)
scenario_ext = scenario.pop() if scenario else "yaml"
print("Processing scenario %s" % scenario_name)
for fname in os.listdir(jobs_dir):
print("Processing %s" % fname)
if fname.startswith(scenario_name):
tags = fname[len(scenario_name):-len(scenario_ext) - 1].split("_")
statuses.append(run_task(os.path.join(jobs_dir, fname), tags))
else:
print("Ignoring file %s" % fname)
print("Exit statuses: %r" % statuses)
template = utils.get_template("ci/index.mako")
with open("rally-plot/extra/index.html", "w") as output:
output.write(template.render())
return any(statuses)
def plot(tasks_results, include_libs=False):
source = _make_source(tasks_results)
tasks = []
subtasks = []
workloads = []
for task in tasks_results:
tasks.append(task)
for subtask in tasks[-1]["subtasks"]:
workloads.extend(subtask.pop("workloads"))
subtasks.extend(tasks[-1].pop("subtasks"))
template = ui_utils.get_template("task/report.html")
data = _process_workloads(workloads)
return template.render(version=version.version_string(),
source=json.dumps(source),
data=json.dumps(data),
include_libs=include_libs)
def generate_report(results):
"""Generates HTML report from test results in JSON format."""
tests = []
for i, name in enumerate(sorted(results["test_cases"])):
test = results["test_cases"][name]
if "tags" in test:
name = "%(name)s [%(tags)s]" % {"name": name,
"tags": ", ".join(test["tags"])}
if "traceback" in test:
output = utils.escape(test["traceback"])
elif "reason" in test:
matcher = SKIP_RE.match(test["reason"])
if matcher:
href = LAUNCHPAD_BUG_LINK.format(matcher.group("bug_number"))
output = re.sub(matcher.group("bug_number"), href,
test["reason"])
else:
output = utils.escape(test["reason"])
else:
output = ""
tests.append({"id": i,
"time": test["time"],
"name": name,
"output": output,
"status": test["status"]})
template = ui_utils.get_template("verification/report.mako")
return template.render(report={
"tests": tests,
"total": results["tests"],
"time": "{0} ({1} s)".format(
datetime.timedelta(seconds=round(
float(results["time"]))), results["time"]),
"success": results["success"],
"failures": results["failures"],
"skipped": results["skipped"],
"expected_failures": results["expected_failures"],
"unexpected_success": results["unexpected_success"]})
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import print_function
import re
import sys
from rally.ui import utils
HELP_MESSAGE = (
"Usage:\n\t"
"render.py ci/template.mako"
"[<key-1>=<value-1> <key-2>=<value-2> ...]\n\n\t"
"Where key-1,value-1 and key-2,value-2 are key pairs of template.")
if __name__ == "__main__":
args = sys.argv
if (len(args) < 1
or not all(re.match("^[^=]+=[^=]+$", arg) for arg in args[2:])):
print(HELP_MESSAGE, file=sys.stderr)
sys.exit(1)
render_kwargs = dict([arg.split("=") for arg in args[2:]])
print(utils.get_template(args[1]).render(**render_kwargs))
def test_get_template(self):
template = utils.get_template("base.html")
self.assertIsInstance(template,
jinja2.environment.Template)
self.assertEqual("base.html", template.name)
self.assertIn("include_raw_file", template.globals)
def test_get_template_mako(self, mock_get_mako_template):
mock_get_mako_template.return_value = "fake_template"
template = utils.get_template("template.mako")
self.assertEqual("fake_template", template)
mock_get_mako_template.assert_called_once_with("template.mako")
def test_get_template_jinja(self, mock_get_jinja_template):
mock_get_jinja_template.return_value = "fake_template"
template = utils.get_template("template.html")
self.assertEqual("fake_template", template)
mock_get_jinja_template.assert_called_once_with("template.html")
def create_report(self):
template = ui_utils.get_template("verification/report.mako")
return template.render(report=self._generate_report())
def plot(results):
template = ui_utils.get_template("task/report.mako")
source, scenarios = _process_results(results)
return template.render(data=json.dumps(scenarios),
source=json.dumps(source))
def test_get_template_jinja(self, mock_get_jinja_template):
mock_get_jinja_template.return_value = "fake_template"
template = utils.get_template("template.html")
self.assertEqual("fake_template", template)
mock_get_jinja_template.assert_called_once_with("template.html")
def to_html(self):
"""Make HTML report."""
template = utils.get_template("verification/report.html")
context = {"uuids": self._uuids, "verifications": self._runs,
"tests": self._tests}
return template.render(data=json.dumps(context), include_libs=False)
def test_get_template_mako(self, mock_get_mako_template):
mock_get_mako_template.return_value = "fake_template"
template = utils.get_template("template.mako")
self.assertEqual("fake_template", template)
mock_get_mako_template.assert_called_once_with("template.mako")
def test_get_template(self):
self.assertIsInstance(utils.get_template("task/report.mako"),
mako.template.Template)
def test_get_template(self):
self.assertIsInstance(utils.get_template("task/report.mako"),
mako.template.Template)
def render_page(**render_vars):
template = utils.get_template("ci/index_verify.html")
with open(os.path.join(BASE_DIR, "extra/index.html"), "w") as f:
f.write(template.render(**render_vars))
def create_report(self):
template = ui_utils.get_template("verification/report.mako")
return template.render(report=self._generate_report())
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import print_function
import re
import sys
from rally.ui import utils
HELP_MESSAGE = (
"Usage:\n\t"
"render.py ci/template.mako"
"[<key-1>=<value-1> <key-2>=<value-2> ...]\n\n\t"
"Where key-1,value-1 and key-2,value-2 are key pairs of template."
)
if __name__ == "__main__":
args = sys.argv
if len(args) < 1 or not all(re.match("^[^=]+=[^=]+$", arg) for arg in args[2:]):
print(HELP_MESSAGE, file=sys.stderr)
sys.exit(1)
render_kwargs = dict([arg.split("=") for arg in args[2:]])
print(utils.get_template(args[1]).render(**render_kwargs))
def render_page(**render_vars):
template = utils.get_template("ci/index_verify.html")
with open(os.path.join(BASE_DIR, "extra/index.html"), "w") as f:
f.write(template.render(**render_vars))
def plot(results):
template = ui_utils.get_template("task/report.mako")
source, scenarios = _process_results(results)
return template.render(data=json.dumps(scenarios),
source=json.dumps(source))
def test_get_template(self, mock_lookup):
mock_lookup.get_template.return_value = "foo_template"
template = utils.get_template("foo_path")
self.assertEqual(template, "foo_template")
mock_lookup.get_template.assert_called_once_with("foo_path")
