def test_task_samples_is_valid(self):
plugins.load()
rally = utils.Rally(force_new_db=True)
db.db_options.set_defaults(db.CONF,
connection="sqlite:///%s/db" %
rally.tmp_dir,
sqlite_db="rally.sqlite")
samples_path = os.path.join(os.path.dirname(__file__), os.pardir,
os.pardir, "samples", "tasks")
matcher = re.compile("\.json$")
if not os.path.exists(utils.DEPLOYMENT_FILE):
subprocess.call(["rally", "deployment", "config"],
stdout=open(utils.DEPLOYMENT_FILE, "w"))
for dirname, dirnames, filenames in os.walk(samples_path):
# NOTE(rvasilets): Skip by suggest of boris-42 because in
# future we don't what to maintain this dir
if dirname.find("tempest-do-not-run-against-production") != -1:
continue
for filename in filenames:
full_path = os.path.join(dirname, filename)
# NOTE(hughsaunders): Skip non config files
# (bug https://bugs.launchpad.net/rally/+bug/1314369)
if not matcher.search(filename):
continue
with open(full_path) as task_file:
try:
input_task = task_file.read()
rendered_task = api.Task.render_template(input_task)
task_config = json.loads(rendered_task)
api.Task.validate("MAIN", task_config)
except Exception as e:
if not self._skip(six.text_type(e)):
print(traceback.format_exc())
print("Failed on task config %s with error." %
full_path)
raise
def test_abort_on_sla_fail(self):
rally = utils.Rally()
cfg = {
"Dummy.dummy_exception": [{
"args": {},
"runner": {
"type": "constant",
"times": 5,
"concurrency": 5
},
"sla": {
"failure_rate": {
"max": 0
}
}
}]
}
config = utils.TaskConfig(cfg)
rally("task start --task %s --abort-on-sla-failure" % config.filename)
expected = [{
"benchmark": "Dummy.dummy_exception",
"criterion": "aborted_on_sla",
"detail": "Task was aborted due to SLA failure(s).",
"pos": 0,
"status": "FAIL"
}, {
"benchmark": "Dummy.dummy_exception",
"criterion": "failure_rate",
"detail": mock.ANY,
"pos": 0,
"status": "FAIL"
}]
try:
rally("task sla_check --json", getjson=True)
except utils.RallyCliError as expected_error:
self.assertEqual(json.loads(expected_error.output), expected)
else:
self.fail("`rally task sla_check` command should return non-zero "
"exit code")
def test_export_with_wrong_connection(self):
rally = utils.Rally()
cfg = {
"Dummy.dummy": [
{
"runner": {
"type": "constant",
"times": 100,
"concurrency": 5
}
}
]
}
config = utils.TaskConfig(cfg)
output = rally("task start --task %s" % config.filename)
uuid = re.search(
r"(?P<uuid>[0-9a-f\-]{36}): started", output).group("uuid")
connection = (
"fake:///" + rally.gen_report_path(extension="json"))
self.assertRaises(utils.RallyCliError,
rally,
"task export --uuid %s --connection %s" % (
uuid, connection))
def test_check_debug(self):
rally = utils.Rally()
rally.env.update(TEST_ENV)
rally("deployment create --name t_create_env --fromenv")
config = rally("deployment config", getjson=True)
config["openstack"]["admin"]["password"] = "******"
file = utils.JsonTempFile(config)
rally("deployment create --name t_create_file_debug "
"--filename %s" % file.filename)
self.assertIn("t_create_file_debug", rally("deployment list"))
self.assertEqual(config, rally("deployment config", getjson=True))
self.assertRaises(utils.RallyCliError, rally, "deployment check")
try:
rally("--debug deployment check")
except utils.RallyCliError as e:
self.assertIn(
"AuthenticationFailed: Unable to establish connection to "
"%s" % TEST_ENV["OS_AUTH_URL"],
str(e))
else:
self.fail("rally deployment fails to raise error for wrong"
" authentication info")
def test_create_from_sysenv(self):
rally = utils.Rally()
rally.env.update(TEST_ENV)
rally("env create --name t_create_env --from-sysenv")
config = rally("env show --only-spec", getjson=True)
self.assertIn("existing@openstack", config)
self.assertEqual(TEST_ENV["OS_USERNAME"],
config["existing@openstack"]["admin"]["username"])
self.assertEqual(TEST_ENV["OS_PASSWORD"],
config["existing@openstack"]["admin"]["password"])
if "project_name" in config["existing@openstack"]["admin"]:
# keystone v3
self.assertEqual(
TEST_ENV["OS_TENANT_NAME"],
config["existing@openstack"]["admin"]["project_name"])
else:
# keystone v2
self.assertEqual(
TEST_ENV["OS_TENANT_NAME"],
config["existing@openstack"]["admin"]["tenant_name"])
self.assertEqual(
TEST_ENV["OS_AUTH_URL"],
config["existing@openstack"]["auth_url"])
def test_sla_success(self):
rally = utils.Rally()
config = utils.TaskConfig(self._get_sample_task_config())
rally("task start --task %s" % config.filename)
rally("task sla_check")
expected = [
{
"benchmark": "KeystoneBasic.create_and_list_users",
"criterion": "max_seconds_per_iteration",
"detail": mock.ANY,
"pos": 0,
"success": True
},
{
"benchmark": "KeystoneBasic.create_and_list_users",
"criterion": "max_failure_percent",
"detail": mock.ANY,
"pos": 0,
"success": True
},
]
data = rally("task sla_check --json", getjson=True)
self.assertEqual(expected, data)
def test_time_hook(self):
rally = utils.Rally()
cfg = self._get_sample_task_config(cmd="/bin/true",
description="event_hook",
runner={
"type": "constant_for_duration",
"concurrency": 3,
"duration": 10
})
cfg["Dummy.dummy"][0]["hooks"].append({
"name": "sys_call",
"description": "time_hook",
"args": "/bin/true",
"trigger": {
"name": "event",
"args": {
"unit": "time",
"at": [3, 6, 9],
}
}
})
config = utils.TaskConfig(cfg)
rally("task start --task %s" % config.filename)
results = json.loads(rally("task results"))
hook_results = results[0]["hooks"]
hooks_cfg = cfg["Dummy.dummy"][0]["hooks"]
expected = [
self._get_result(hooks_cfg[0], iterations=[5]),
self._get_result(hooks_cfg[1], seconds=[3, 6, 9])
]
self.assertEqual(
expected,
sorted(hook_results,
key=lambda i: i["config"]["trigger"]["args"]["unit"]))
self._assert_results_time(hook_results)
def test_validate_with_plugin_paths(self):
rally = utils.Rally()
plugin_paths = ("tests/functional/extra/fake_dir1/,"
"tests/functional/extra/fake_dir2/")
task_file = "tests/functional/extra/test_fake_scenario.json"
output = rally(("--plugin-paths %(plugin_paths)s "
"task validate --task %(task_file)s") % {
"task_file": task_file,
"plugin_paths": plugin_paths
})
self.assertIn("Task config is valid", output)
plugin_paths = ("tests/functional/extra/fake_dir1/"
"fake_plugin1.py,"
"tests/functional/extra/fake_dir2/"
"fake_plugin2.py")
task_file = "tests/functional/extra/test_fake_scenario.json"
output = rally(("--plugin-paths %(plugin_paths)s "
"task validate --task %(task_file)s") % {
"task_file": task_file,
"plugin_paths": plugin_paths
})
self.assertIn("Task config is valid", output)
plugin_paths = ("tests/functional/extra/fake_dir1/,"
"tests/functional/extra/fake_dir2/"
"fake_plugin2.py")
task_file = "tests/functional/extra/test_fake_scenario.json"
output = rally(("--plugin-paths %(plugin_paths)s "
"task validate --task %(task_file)s") % {
"task_file": task_file,
"plugin_paths": plugin_paths
})
self.assertIn("Task config is valid", output)
def test_start_with_empty_config(self):
rally = utils.Rally()
config = utils.TaskConfig(None)
with self.assertRaises(utils.RallyCliError) as err:
rally("task start --task %s" % config.filename)
self.assertIn("Input task is empty", err.exception.output)
def setUp(self):
super(PluginTestCase, self).setUp()
self.rally = utils.Rally()
def test_status(self):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
rally("task start --task %s" % config.filename)
self.assertIn("finished", rally("task status"))
def test_check_fail(self):
rally = utils.Rally()
rally.env.update(TEST_ENV)
rally("deployment create --name t_create_env --fromenv")
self.assertRaises(utils.RallyCliError, rally, "deployment check")
def test_list_plugins(self):
rally = utils.Rally(plugin_path="tests/functional/extra")
output = rally("verify list-plugins")
self.assertIn("fakeverifier", output)
self.assertIn("installation", output)
def test_list(self):
rally = utils.Rally()
result = rally("plugin list Dummy")
self.assertIn("Dummy.dummy", result)
self.assertIn("Dummy.dummy_exception", result)
self.assertIn("Dummy.dummy_random_fail_in_atomic", result)
def test_check_success(self):
rally = utils.Rally()
rally("deployment check")
def test_check_success(self):
rally = utils.Rally()
rally("env check")
def test_check_success(self):
rally = utils.Rally()
self.assertTrue(rally("deployment check"))
def test_report_with_wrong_task_id(self):
rally = utils.Rally()
self.assertRaises(utils.RallyCliError,
rally, "task report --tasks %s" % FAKE_TASK_UUID)
self.assertRaises(utils.RallyCliError,
rally, "task report --uuid %s" % FAKE_TASK_UUID)
def test_validate_is_valid(self):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
output = rally("task validate --task %s" % config.filename)
self.assertIn("Task config is valid", output)
def setUp(self):
super(VerifyTestCase, self).setUp()
self.rally = utils.Rally()
def test_list_not_found_name(self):
rally = utils.Rally()
result = rally("plugin list Dummy2222")
self.assertIn("Plugin Dummy2222 not found", result)
def test_list_not_found_platform(self):
rally = utils.Rally()
result = rally("plugin list --platform some")
self.assertIn("Platform some not found", result)
def test_results(self):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
rally("task start --task %s" % config.filename)
self.assertIn("result", rally("task results"))
def test_create_empty(self):
rally = utils.Rally()
rally("deployment create --name t_empty")
self.assertEqual("{}", rally("deployment config").strip())
def test_detailed_with_wrong_task_id(self):
rally = utils.Rally()
self.assertRaises(utils.RallyCliError,
rally, "task detailed --uuid %s" % FAKE_TASK_UUID)
def test_show_not_found(self):
rally = utils.Rally()
name = "Dummy666666"
result = self.assertRaises(utils.RallyCliError, rally,
"plugin show %s" % name)
self.assertIn("Plugin %s not found" % name, result.output)
def test_status_with_wrong_task_id(self):
rally = utils.Rally()
self.assertRaises(utils.RallyCliError,
rally, "task status --uuid %s" % FAKE_TASK_UUID)
def setUp(self):
super(DeploymentTestCase, self).setUp()
self.rally = utils.Rally()
def test_sla_fail(self):
rally = utils.Rally()
cfg = self._get_sample_task_config(max_seconds_per_iteration=0.001)
config = utils.TaskConfig(cfg)
rally("task start --task %s" % config.filename)
self.assertRaises(utils.RallyCliError, rally, "task sla-check")
def test_recreate(self):
rally = utils.Rally()
rally.env.update(utils.TEST_ENV)
rally("deployment create --name t_create_env --fromenv")
rally("deployment recreate --deployment t_create_env")
self.assertIn("t_create_env", rally("deployment list"))
