def test_schema_is_valid(self, mock_benchmark_engine__validate_config_semantic): scenarios = set() for dirname, dirnames, filenames in os.walk(self.samples_path): for filename in filenames: full_path = os.path.join(dirname, filename) # NOTE(hughsaunders): Skip non config files # (bug https://bugs.launchpad.net/rally/+bug/1314369) if not re.search("\.(ya?ml|json)$", filename, flags=re.I): continue with open(full_path) as task_file: try: task_config = yaml.safe_load(api.Task.render_template (task_file.read())) eng = engine.BenchmarkEngine(task_config, mock.MagicMock()) eng.validate() except Exception: print(traceback.format_exc()) self.fail("Invalid task file: %s" % full_path) else: scenarios.update(task_config.keys()) missing = set(s.get_name() for s in scenario.Scenario.get_all()) missing -= scenarios # check missing scenario is not from plugin missing = [s for s in list(missing) if scenario.Scenario.get(s).__module__.startswith("rally")] self.assertEqual(missing, [], "These scenarios don't have samples: %s" % missing)
def start(cls, deployment, config, task=None, abort_on_sla_failure=False): """Start a task. Task is a list of benchmarks that will be called one by one, results of execution will be stored in DB. :param deployment: UUID or name of the deployment :param config: a dict with a task configuration :param task: Task object. If None, it will be created :param abort_on_sla_failure: if True, the execution of a benchmark scenario will stop when any SLA check for it fails """ deployment = objects.Deployment.get(deployment) task = task or objects.Task(deployment_uuid=deployment["uuid"]) if task.is_temporary: raise ValueError( _("Unable to run a temporary task. Please check your code.")) LOG.info("Benchmark Task %s on Deployment %s" % (task["uuid"], deployment["uuid"])) benchmark_engine = engine.BenchmarkEngine( config, task, admin=deployment["admin"], users=deployment["users"], abort_on_sla_failure=abort_on_sla_failure) try: benchmark_engine.run() except Exception: deployment.update_status(consts.DeployStatus.DEPLOY_INCONSISTENT) raise
def test__prepare_context(self, mock_scenario_get): default_context = {"a": 1, "b": 2} mock_scenario_get.return_value._meta_get.return_value = default_context task = mock.MagicMock() name = "a.benchmark" context = {"b": 3, "c": 4} endpoint = mock.MagicMock() config = { "a.benchmark": [{ "context": { "context_a": { "a": 1 } } }], } eng = engine.BenchmarkEngine(config, task) result = eng._prepare_context(context, name, endpoint) expected_context = copy.deepcopy(default_context) expected_context.setdefault("users", {}) expected_context.update(context) expected_result = { "task": task, "admin": { "endpoint": endpoint }, "scenario_name": name, "config": expected_context } self.assertEqual(result, expected_result) mock_scenario_get.assert_called_once_with(name) mock_scenario_get.return_value._meta_get.assert_called_once_with( "default_context")
def test__validate_config_semantic( self, mock_deployment_get, mock__validate_config_semantic_helper, mock_users_ctx, mock_clients): mock_users_ctx.UserGenerator = fakes.FakeUserContext mock_clients.return_value = mock.MagicMock() config = { "a": [mock.MagicMock(), mock.MagicMock()], "b": [mock.MagicMock()] } fake_task = mock.MagicMock() eng = engine.BenchmarkEngine(config, fake_task) eng.admin = "admin" eng._validate_config_semantic(config) expected_calls = [ mock.call("admin"), mock.call(fakes.FakeUserContext.user["endpoint"]) ] mock_clients.assert_has_calls(expected_calls) mock_deployment_get.assert_called_once_with(fake_task["uuid"]) admin = user = mock_clients.return_value fake_deployment = mock_deployment_get.return_value expected_calls = [ mock.call(admin, user, "a", 0, fake_deployment, config["a"][0]), mock.call(admin, user, "a", 1, fake_deployment, config["a"][1]), mock.call(admin, user, "b", 0, fake_deployment, config["b"][0]) ] mock__validate_config_semantic_helper.assert_has_calls( expected_calls, any_order=True)
def test__prepare_context_with_existing_users(self, mock_scenario_get): mock_scenario_get.return_value._meta_get.return_value = {} task = mock.MagicMock() name = "a.benchmark" context = {"b": 3, "c": 4} endpoint = mock.MagicMock() config = { "a.benchmark": [{ "context": { "context_a": { "a": 1 } } }], } existing_users = [mock.MagicMock()] eng = engine.BenchmarkEngine(config, task, users=existing_users) result = eng._prepare_context(context, name, endpoint) expected_context = {"existing_users": existing_users} expected_context.update(context) expected_result = { "task": task, "admin": { "endpoint": endpoint }, "scenario_name": name, "config": expected_context } self.assertEqual(result, expected_result) mock_scenario_get.assert_called_once_with(name) mock_scenario_get.return_value._meta_get.assert_called_once_with( "default_context")
def test_schema_is_valid(self, mock_benchmark_engine__validate_config_semantic): scenarios = set() for dirname, dirnames, filenames in os.walk(self.samples_path): for filename in filenames: full_path = os.path.join(dirname, filename) # NOTE(hughsaunders): Skip non config files # (bug https://bugs.launchpad.net/rally/+bug/1314369) if not re.search("\.(ya?ml|json)$", filename, flags=re.I): continue with open(full_path) as task_file: try: task_config = yaml.safe_load(task_file.read()) eng = engine.BenchmarkEngine(task_config, mock.MagicMock()) eng.validate() except Exception: print(traceback.format_exc()) self.fail("Invalid task file: %s" % full_path) else: scenarios.update(task_config.keys()) # TODO(boris-42): We should refactor scenarios framework add "_" to # all non-benchmark methods.. Then this test will pass. missing = set(scenario.Scenario.list_benchmark_scenarios()) - scenarios # check missing scenario is not from plugin missing = [ s for s in list(missing) if scenario.Scenario.get_by_name( s.split(".")[0]).__module__.startswith("rally") ] self.assertEqual(missing, [], "These scenarios don't have samples: %s" % missing)
def test_schema_is_valid(self, mock_benchmark_engine__validate_config_semantic): discover.load_plugins(os.path.join(self.rally_jobs_path, "plugins")) for filename in [ "rally.yaml", "rally-neutron.yaml", "rally-zaqar.yaml", "rally-designate.yaml" ]: full_path = os.path.join(self.rally_jobs_path, filename) with open(full_path) as task_file: try: args_file = os.path.join( self.rally_jobs_path, filename.rsplit(".", 1)[0] + "_args.yaml") args = {} if os.path.exists(args_file): args = yaml.safe_load(open(args_file).read()) if not isinstance(args, dict): raise TypeError( "args file %s must be dict in yaml or json " "presenatation" % args_file) task = api.Task.render_template(task_file.read(), **args) task = yaml.safe_load(task) eng = engine.BenchmarkEngine(task, mock.MagicMock()) eng.validate() except Exception: print(traceback.format_exc()) self.fail("Wrong task input file: %s" % full_path)
def test_run_exception_is_logged(self, mock_context_manager_setup, mock_context_manager_cleanup, mock_scenario_runner, mock_scenario, mock_result_consumer, mock_log, mock_task_get_status): mock_context_manager_setup.side_effect = Exception mock_result_consumer.is_task_in_aborting_status.return_value = False config = { "a.benchmark": [{ "context": { "context_a": { "a": 1 } } }], "b.benchmark": [{ "context": { "context_b": { "b": 2 } } }] } task = mock.MagicMock() eng = engine.BenchmarkEngine(config, task) eng.run() self.assertEqual(2, mock_log.exception.call_count)
def test_run__task_aborted(self, mock_scenario_runner, mock_scenario, mock_context_manager_setup, mock_context_manager_cleanup, mock_result_consumer, mock_task_get_status): task = mock.MagicMock() config = { "a.benchmark": [{ "runner": { "type": "a", "b": 1 } }], "b.benchmark": [{ "runner": { "type": "a", "b": 1 } }], "c.benchmark": [{ "runner": { "type": "a", "b": 1 } }] } fake_runner_cls = mock.MagicMock() fake_runner = mock.MagicMock() fake_runner_cls.return_value = fake_runner mock_task_get_status.return_value = consts.TaskStatus.SOFT_ABORTING mock_scenario_runner.get.return_value = fake_runner_cls eng = engine.BenchmarkEngine(config, task) eng.run() self.assertEqual(mock.call(consts.TaskStatus.ABORTED), task.update_status.mock_calls[-1])
def test_run__config_has_context(self, mock_context_manager_setup, mock_context_manager_cleanup, mock_scenario_runner, mock_scenario, mock_result_consumer, mock_task_get_status): config = { "a.benchmark": [{ "context": { "context_a": { "a": 1 } } }], "b.benchmark": [{ "context": { "context_b": { "b": 2 } } }] } task = mock.MagicMock() mock_task_get_status.return_value = consts.TaskStatus.RUNNING eng = engine.BenchmarkEngine(config, task) eng.run()
def test__validate_config_semanitc_helper_invalid_arg( self, mock_scenario_validate): eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock()) self.assertRaises(exceptions.InvalidBenchmarkConfig, eng._validate_config_semantic_helper, "a", "u", "n", "p", mock.MagicMock(), {})
def start(cls, deployment, config, task=None, abort_on_sla_failure=False): """Start a task. Task is a list of benchmarks that will be called one by one, results of execution will be stored in DB. :param deployment: UUID or name of the deployment :param config: a dict with a task configuration :param task: Task object. If None, it will be created :param abort_on_sla_failure: if True, the execution of a benchmark scenario will stop when any SLA check for it fails """ deployment = objects.Deployment.get(deployment) task = task or objects.Task(deployment_uuid=deployment["uuid"]) LOG.info("Benchmark Task %s on Deployment %s" % (task["uuid"], deployment["uuid"])) benchmark_engine = engine.BenchmarkEngine( config, task, admin=deployment["admin"], users=deployment["users"], abort_on_sla_failure=abort_on_sla_failure) try: benchmark_engine.validate() benchmark_engine.run() except exceptions.InvalidTaskException: # NOTE(boris-42): We don't log anything, because it's a normal # situation when a user puts a wrong config. pass except Exception: deployment.update_status(consts.DeployStatus.DEPLOY_INCONSISTENT) raise
def test__validate_config_scenarios_name_non_exsisting( self, mock_scenario): config = {"exist": [], "nonexist1": [], "nonexist2": []} mock_scenario.list_benchmark_scenarios.return_value = ["exist", "aaa"] eng = engine.BenchmarkEngine(config, mock.MagicMock()) self.assertRaises(exceptions.NotFoundScenarios, eng._validate_config_scenarios_name, config)
def test_init(self, mock_task_config): config = mock.MagicMock() task = mock.MagicMock() mock_task_config.return_value = fake_task_instance = mock.MagicMock() eng = engine.BenchmarkEngine(config, task) mock_task_config.assert_has_calls([mock.call(config)]) self.assertEqual(eng.config, fake_task_instance) self.assertEqual(eng.task, task)
def test_validate__wrong_scenarios_name(self, mock_validate): task = mock.MagicMock() eng = engine.BenchmarkEngine(mock.MagicMock(), task) eng._validate_config_scenarios_name = mock.MagicMock( side_effect=exceptions.NotFoundScenarios) self.assertRaises(exceptions.InvalidTaskException, eng.validate) self.assertTrue(task.set_failed.called)
def test__validate_config_semantic_helper(self, mock_scenario_validate): deployment = mock.MagicMock() eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock()) eng._validate_config_semantic_helper("admin", "user", "name", "pos", deployment, {"args": "args"}) mock_scenario_validate.assert_called_once_with( "name", {"args": "args"}, admin="admin", users=["user"], deployment=deployment)
def test__validate_config_scenarios_name(self, mock_scenario): config = { "a": [], "b": [] } mock_scenario.list_benchmark_scenarios.return_value = ["e", "b", "a"] eng = engine.BenchmarkEngine(config, mock.MagicMock()) eng._validate_config_scenarios_name(config)
def test__validate_config_syntax__wrong_context( self, mock_context_manager, mock_scenario_runner_validate): config = {"sca": [{"context": "a"}], "scb": [{"runner": "b"}]} eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock()) mock_context_manager.validate = mock.MagicMock( side_effect=jsonschema.ValidationError("a")) self.assertRaises(exceptions.InvalidBenchmarkConfig, eng._validate_config_syntax, config)
def test_validate__wrong_schema(self): config = { "wrong": True } task = mock.MagicMock() eng = engine.BenchmarkEngine(config, task) self.assertRaises(exceptions.InvalidTaskException, eng.validate) self.assertTrue(task.set_failed.called)
def test_validate__wrong_semantic(self, mock_task_config): task = mock.MagicMock() eng = engine.BenchmarkEngine(mock.MagicMock(), task) eng._validate_config_scenarios_name = mock.MagicMock() eng._validate_config_syntax = mock.MagicMock() eng._validate_config_semantic = mock.MagicMock( side_effect=exceptions.InvalidBenchmarkConfig) self.assertRaises(exceptions.InvalidTaskException, eng.validate) self.assertTrue(task.set_failed.called)
def test_run__config_has_context( self, mock_context_manager_setup, mock_context_manager_cleanup, mock_scenario_runner, mock_scenario, mock_result_consumer): config = { "a.benchmark": [{"context": {"context_a": {"a": 1}}}], "b.benchmark": [{"context": {"context_b": {"b": 2}}}] } task = mock.MagicMock() eng = engine.BenchmarkEngine(config, task) eng.run()
def test__validate_config_scenarios_name(self, mock_scenario_get_all): config = {"a": [], "b": []} mock_scenario_get_all.return_value = [ mock.MagicMock(get_name=lambda: "e"), mock.MagicMock(get_name=lambda: "b"), mock.MagicMock(get_name=lambda: "a") ] eng = engine.BenchmarkEngine(config, mock.MagicMock()) eng._validate_config_scenarios_name(config)
def test__validate_config_syntax( self, mock_context_manager_validate, mock_scenario_runner_validate): config = {"sca": [{"context": "a"}], "scb": [{"runner": "b"}]} eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock()) eng._validate_config_syntax(config) mock_scenario_runner_validate.assert_has_calls( [mock.call({}), mock.call("b")], any_order=True) mock_context_manager_validate.assert_has_calls( [mock.call("a", non_hidden=True), mock.call({}, non_hidden=True)], any_order=True)
def test_run__update_status( self, mock_scenario_runner, mock_scenario, mock_context_manager_setup, mock_context_manager_cleanup, mock_result_consumer): task = mock.MagicMock() eng = engine.BenchmarkEngine([], task) eng.run() task.update_status.assert_has_calls([ mock.call(consts.TaskStatus.RUNNING), mock.call(consts.TaskStatus.FINISHED) ])
def validate(cls, deployment, config): """Validate a task config against specified deployment. :param deployment: UUID or name of the deployment :param config: a dict with a task configuration """ deployment = objects.Deployment.get(deployment) task = objects.Task(deployment_uuid=deployment["uuid"], fake=True) benchmark_engine = engine.BenchmarkEngine(config, task, admin=deployment["admin"], users=deployment["users"]) benchmark_engine.validate()
def test_get_user_ctx_for_validation_existing_users( self, mock_existing_users): context = {"a": 10} users = [mock.MagicMock(), mock.MagicMock()] eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock(), users=users) result = eng._get_user_ctx_for_validation(context) self.assertEqual(context["config"]["existing_users"], users) mock_existing_users.assert_called_once_with(context) self.assertEqual(mock_existing_users.return_value, result)
def test__validate_config_syntax__wrong_context( self, mock_context_manager, mock_scenario_runner_validate, mock_task_config): mock_task_instance = mock.MagicMock() mock_subtask = mock.MagicMock() mock_subtask.scenarios = [ {"name": "sca", "context": "a"}, {"name": "scb", "runner": "b"} ] mock_task_instance.subtasks = [mock_subtask] eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock()) mock_context_manager.validate = mock.MagicMock( side_effect=jsonschema.ValidationError("a")) self.assertRaises(exceptions.InvalidBenchmarkConfig, eng._validate_config_syntax, mock_task_instance)
def test__validate_config_semantic( self, mock_deployment_get, mock__validate_config_semantic_helper, mock_users_ctx, mock_clients, mock_task_config): mock_users_ctx.UserGenerator = fakes.FakeUserContext mock_clients.return_value = mock.MagicMock() mock_task_instance = mock.MagicMock() mock_subtask1 = mock.MagicMock() mock_subtask1.scenarios = [ {"name": "a", "kw": 0}, {"name": "a", "kw": 1} ] mock_subtask2 = mock.MagicMock() mock_subtask2.scenarios = [ {"name": "b", "kw": 0}, ] mock_task_instance.subtasks = [mock_subtask1, mock_subtask2] fake_task = mock.MagicMock() eng = engine.BenchmarkEngine(mock_task_instance, fake_task) eng.admin = "admin" eng._validate_config_semantic(mock_task_instance) expected_calls = [ mock.call("admin"), mock.call(fakes.FakeUserContext.user["endpoint"]) ] mock_clients.assert_has_calls(expected_calls) mock_deployment_get.assert_called_once_with(fake_task["uuid"]) admin = user = mock_clients.return_value fake_deployment = mock_deployment_get.return_value expected_calls = [ mock.call(admin, user, "a", 0, fake_deployment, {"name": "a", "kw": 0}), mock.call(admin, user, "a", 1, fake_deployment, {"name": "a", "kw": 1}), mock.call(admin, user, "b", 0, fake_deployment, {"name": "b", "kw": 0}) ] mock__validate_config_semantic_helper.assert_has_calls( expected_calls, any_order=True)
def test__validate_config_scenarios_name_non_exsisting( self, mock_scenario, mock_task_config): mock_task_instance = mock.MagicMock() mock_subtask = mock.MagicMock() mock_subtask.scenarios = [ {"name": "exist"}, {"name": "nonexist1"}, {"name": "nonexist2"} ] mock_task_instance.subtasks = [mock_subtask] mock_scenario.list_benchmark_scenarios.return_value = ["exist", "aaa"] eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock()) self.assertRaises(exceptions.NotFoundScenarios, eng._validate_config_scenarios_name, mock_task_instance)
def test_validate(self, mock_validate): config = mock.MagicMock() eng = engine.BenchmarkEngine(config, mock.MagicMock()) mock_validate = mock.MagicMock() eng._validate_config_scenarios_name = mock_validate.names eng._validate_config_syntax = mock_validate.syntax eng._validate_config_semantic = mock_validate.semantic eng.validate() expected_calls = [ mock.call.names(config), mock.call.syntax(config), mock.call.semantic(config) ] mock_validate.assert_has_calls(expected_calls)