def test_run_job_execution(self, mock_clients):
mock_clients("sahara").job_executions.get.side_effect = [
mock.MagicMock(info={"status": "pending"}, id="42"),
mock.MagicMock(info={"status": "SUCCESS"}, id="42")
]
mock_clients("sahara").job_executions.create.return_value = (
mock.MagicMock(id="42"))
scenario = utils.SaharaScenario()
scenario._run_job_execution(job_id="test_job_id",
cluster_id="test_cluster_id",
input_id="test_input_id",
output_id="test_output_id",
configs={"k": "v"},
job_idx=0)
mock_clients("sahara").job_executions.create.assert_called_once_with(
job_id="test_job_id",
cluster_id="test_cluster_id",
input_id="test_input_id",
output_id="test_output_id",
configs={"k": "v"})
je_get_expected = mock.call("42")
mock_clients("sahara").job_executions.get.assert_has_calls(
[je_get_expected, je_get_expected])
def test_run_job_execution_fail(self, mock_clients):
mock_clients("sahara").job_executions.get.side_effect = [
mock.MagicMock(info={"status": "pending"}, id="42"),
mock.MagicMock(info={"status": "killed"}, id="42")
]
mock_clients("sahara").job_executions.create.return_value = (
mock.MagicMock(id="42"))
scenario = utils.SaharaScenario()
self.assertRaises(exceptions.RallyException,
scenario._run_job_execution,
job_id="test_job_id",
cluster_id="test_cluster_id",
input_id="test_input_id",
output_id="test_output_id",
configs={"k": "v"},
job_idx=0)
mock_clients("sahara").job_executions.create.assert_called_once_with(
job_id="test_job_id",
cluster_id="test_cluster_id",
input_id="test_input_id",
output_id="test_output_id",
configs={"k": "v"})
def test_list_node_group_templates(self, mock_clients):
ngts = []
mock_clients("sahara").node_group_templates.list.return_value = ngts
scenario = utils.SaharaScenario()
return_ngts_list = scenario._list_node_group_templates()
self.assertEqual(ngts, return_ngts_list)
self._test_atomic_action_timer(scenario.atomic_actions(),
'sahara.list_node_group_templates')
def test_delete_node_group_templates(self, mock_clients):
scenario = utils.SaharaScenario()
ng = mock.MagicMock(id=42)
scenario._delete_node_group_template(ng)
delete_mock = mock_clients("sahara").node_group_templates.delete
delete_mock.assert_called_once_with(42)
self._test_atomic_action_timer(scenario.atomic_actions(),
'sahara.delete_node_group_template')
def test_create_output_ds_swift(self, mock_clients, mock_random_name):
ctxt = {
"sahara_output_conf": {
"output_type": "swift",
"output_url_prefix": "swift://test_out/"
}
}
scenario = utils.SaharaScenario(ctxt)
self.assertRaises(exceptions.RallyException,
scenario._create_output_ds)
def test_create_node_group_templates(self, mock_clients, mock_random_name):
scenario = utils.SaharaScenario()
mock_processes = {
"test_plugin": {
"test_version": {
"master": ["p1"],
"worker": ["p2"]
}
}
}
scenario.NODE_PROCESSES = mock_processes
scenario._create_master_node_group_template(
flavor_id="test_flavor",
plugin_name="test_plugin",
hadoop_version="test_version"
)
scenario._create_worker_node_group_template(
flavor_id="test_flavor",
plugin_name="test_plugin",
hadoop_version="test_version"
)
create_calls = [
mock.call(
name="random_name",
plugin_name="test_plugin",
hadoop_version="test_version",
flavor_id="test_flavor",
node_processes=["p1"]),
mock.call(
name="random_name",
plugin_name="test_plugin",
hadoop_version="test_version",
flavor_id="test_flavor",
node_processes=["p2"]
)]
mock_clients("sahara").node_group_templates.create.assert_has_calls(
create_calls)
self._test_atomic_action_timer(
scenario.atomic_actions(),
"sahara.create_master_node_group_template")
self._test_atomic_action_timer(
scenario.atomic_actions(),
"sahara.create_worker_node_group_template")
def test_create_output_ds(self, mock_clients, mock_random_name):
ctxt = {
"sahara_output_conf": {
"output_type": "hdfs",
"output_url_prefix": "hdfs://test_out/"
}
}
scenario = utils.SaharaScenario(ctxt)
scenario._create_output_ds()
mock_clients("sahara").data_sources.create.assert_called_once_with(
name="42",
description="",
data_source_type="hdfs",
url="hdfs://test_out/42")
def test_launch_cluster_error(self, mock_constants, mock_clients,
mock_random_name):
scenario = utils.SaharaScenario(clients=mock.MagicMock())
mock_processes = {
"test_plugin": {
"test_version": {
"master": ["p1"],
"worker": ["p2"]
}
}
}
mock_configs = {
"test_plugin": {
"test_version": {
"target": "HDFS",
"config_name": "dfs.replication"
}
}
}
mock_constants.NODE_PROCESSES = mock_processes
mock_constants.REPLICATION_CONFIGS = mock_configs
mock_clients("sahara").clusters.create.return_value = mock.MagicMock(
id="test_cluster_id")
mock_clients("sahara").clusters.get.return_value = mock.MagicMock(
status="error")
self.assertRaises(
exceptions.SaharaClusterFailure,
scenario._launch_cluster,
plugin_name="test_plugin",
hadoop_version="test_version",
flavor_id="test_flavor",
image_id="test_image",
floating_ip_pool="test_pool",
volumes_per_node=5,
volumes_size=10,
workers_count=42,
node_configs={"HDFS": {
"local_config": "local_value"
}})
def test_delete_cluster(self, mock_clients):
scenario = utils.SaharaScenario()
cluster = mock.MagicMock(id=42)
mock_clients("sahara").clusters.get.side_effect = [
cluster, sahara_base.APIException()
]
scenario._delete_cluster(cluster)
delete_mock = mock_clients("sahara").clusters.delete
delete_mock.assert_called_once_with(42)
cl_get_expected = mock.call(42)
mock_clients("sahara").clusters.get.assert_has_calls(
[cl_get_expected, cl_get_expected])
self._test_atomic_action_timer(scenario.atomic_actions(),
'sahara.delete_cluster')
def setup(self):
wait_dict = dict()
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["endpoint"])
image_id = self.context["tenants"][tenant_id]["sahara_image"]
floating_ip_pool = self.config.get("floating_ip_pool")
temporary_context = {"tenant": self.context["tenants"][tenant_id]}
cluster = utils.SaharaScenario(
context=temporary_context, clients=clients)._launch_cluster(
plugin_name=self.config["plugin_name"],
hadoop_version=self.config["hadoop_version"],
flavor_id=self.config["flavor_id"],
workers_count=self.config["workers_count"],
image_id=image_id,
floating_ip_pool=floating_ip_pool,
volumes_per_node=self.config.get("volumes_per_node"),
volumes_size=self.config.get("volumes_size", 1),
auto_security_group=self.config.get(
"auto_security_group", True),
security_groups=self.config.get("security_groups"),
node_configs=self.config.get("node_configs"),
cluster_configs=self.config.get("cluster_configs"),
enable_anti_affinity=self.config.get(
"enable_anti_affinity", False),
wait_active=False)
self.context["tenants"][tenant_id]["sahara_cluster"] = cluster.id
# Need to save the client instance to poll for active status
wait_dict[cluster] = clients.sahara()
bench_utils.wait_for(
resource=wait_dict,
update_resource=self.update_clusters_dict,
is_ready=self.all_clusters_active,
timeout=CONF.benchmark.cluster_create_timeout,
check_interval=CONF.benchmark.cluster_check_interval)
def test_scale_cluster(self, mock_clients):
scenario = utils.SaharaScenario()
cluster = mock.MagicMock(id=42,
node_groups=[{
"name": "random_master",
"count": 1
}, {
"name": "random_worker",
"count": 41
}])
mock_clients("sahara").clusters.get.return_value = mock.MagicMock(
id=42, status="active")
expected_scale_object = {
"resize_node_groups": [{
"name": "random_worker",
"count": 42
}]
}
scenario._scale_cluster(cluster, 1)
mock_clients("sahara").clusters.scale.assert_called_once_with(
42, expected_scale_object)
def test_launch_cluster(self, mock_clients, mock_random_name):
scenario = utils.SaharaScenario()
mock_processes = {
"test_plugin": {
"test_version": {
"master": ["p1"],
"worker": ["p2"]
}
}
}
mock_configs = {
"test_plugin": {
"test_version": {
"target": "HDFS",
"config_name": "dfs.replication"
}
}
}
node_groups = [{
"name": "master-ng",
"flavor_id": "test_flavor",
"node_processes": ["p1"],
"floating_ip_pool": "test_pool",
"volumes_per_node": 5,
"volumes_size": 10,
"count": 1,
"node_configs": {
"HDFS": {
"local_config": "local_value"
}
}
}, {
"name": "worker-ng",
"flavor_id": "test_flavor",
"node_processes": ["p2"],
"floating_ip_pool": "test_pool",
"volumes_per_node": 5,
"volumes_size": 10,
"count": 41,
"node_configs": {
"HDFS": {
"local_config": "local_value"
}
}
}]
scenario.NODE_PROCESSES = mock_processes
scenario.REPLICATION_CONFIGS = mock_configs
mock_clients("sahara").clusters.create.return_value = mock.MagicMock(
id="test_cluster_id")
mock_clients("sahara").clusters.get.return_value = mock.MagicMock(
status="active")
scenario._launch_cluster(
plugin_name="test_plugin",
hadoop_version="test_version",
flavor_id="test_flavor",
image_id="test_image",
floating_ip_pool="test_pool",
volumes_per_node=5,
volumes_size=10,
node_count=42,
node_configs={"HDFS": {
"local_config": "local_value"
}})
mock_clients("sahara").clusters.create.assert_called_once_with(
name="random_name",
plugin_name="test_plugin",
hadoop_version="test_version",
node_groups=node_groups,
default_image_id="test_image",
cluster_configs={"HDFS": {
"dfs.replication": 3
}},
net_id=None)
self._test_atomic_action_timer(scenario.atomic_actions(),
'sahara.launch_cluster')
def test_launch_cluster(self, mock_constants, mock_clients,
mock_random_name):
clients_values = mock.MagicMock(return_value=[consts.Service.NEUTRON])
mock_clients.services.return_value = mock.MagicMock(
values=clients_values)
context = {
"tenant": {
"networks": [{
"id": "test_neutron_id",
"router_id": "test_router_id"
}]
}
}
scenario = utils.SaharaScenario(context=context, clients=mock_clients)
mock_processes = {
"test_plugin": {
"test_version": {
"master": ["p1"],
"worker": ["p2"]
}
}
}
mock_configs = {
"test_plugin": {
"test_version": {
"target": "HDFS",
"config_name": "dfs.replication"
}
}
}
floating_ip_pool_uuid = uuidutils.generate_uuid()
node_groups = [{
"name": "master-ng",
"flavor_id": "test_flavor",
"node_processes": ["p1"],
"floating_ip_pool": floating_ip_pool_uuid,
"volumes_per_node": 5,
"volumes_size": 10,
"count": 1,
"auto_security_group": True,
"security_groups": ["g1", "g2"],
"node_configs": {
"HDFS": {
"local_config": "local_value"
}
},
}, {
"name": "worker-ng",
"flavor_id": "test_flavor",
"node_processes": ["p2"],
"floating_ip_pool": floating_ip_pool_uuid,
"volumes_per_node": 5,
"volumes_size": 10,
"count": 42,
"auto_security_group": True,
"security_groups": ["g1", "g2"],
"node_configs": {
"HDFS": {
"local_config": "local_value"
}
},
}]
mock_constants.NODE_PROCESSES = mock_processes
mock_constants.REPLICATION_CONFIGS = mock_configs
mock_clients("sahara").clusters.create.return_value = mock.MagicMock(
id="test_cluster_id")
mock_clients("sahara").clusters.get.return_value = mock.MagicMock(
status="active")
scenario._launch_cluster(
plugin_name="test_plugin",
hadoop_version="test_version",
flavor_id="test_flavor",
image_id="test_image",
floating_ip_pool=floating_ip_pool_uuid,
volumes_per_node=5,
volumes_size=10,
auto_security_group=True,
security_groups=["g1", "g2"],
workers_count=42,
node_configs={"HDFS": {
"local_config": "local_value"
}})
mock_clients("sahara").clusters.create.assert_called_once_with(
name="random_name",
plugin_name="test_plugin",
hadoop_version="test_version",
node_groups=node_groups,
default_image_id="test_image",
cluster_configs={"HDFS": {
"dfs.replication": 3
}},
net_id="test_neutron_id",
anti_affinity=None)
self._test_atomic_action_timer(scenario.atomic_actions(),
"sahara.launch_cluster")
def setup(self):
ready_tenants = set()
wait_dict = dict()
for user in self.context.get("users", []):
tenant_id = user["tenant_id"]
if tenant_id not in ready_tenants:
ready_tenants.add(tenant_id)
clients = osclients.Clients(user["endpoint"])
image_id = self.context["sahara_images"][tenant_id]
neutron_net = self.config.get("neutron_net")
if not neutron_net:
# Skipping fixed network config
neutron_net_id = None
else:
network_cfg = {"name": neutron_net}
neutron_net_id = (types.NeutronNetworkResourceType
.transform(clients, network_cfg))
floating_ip_pool = self.config.get("floating_ip_pool")
if not floating_ip_pool:
# Skipping floating network config
floating_ip_pool_id = None
else:
network_cfg = {"name": floating_ip_pool}
floating_ip_pool_id = (types.NeutronNetworkResourceType
.transform(clients, network_cfg))
cluster = utils.SaharaScenario(
context=self.context, clients=clients)._launch_cluster(
plugin_name=self.config["plugin_name"],
hadoop_version=self.config["hadoop_version"],
flavor_id=self.config["flavor_id"],
node_count=self.config["node_count"],
image_id=image_id,
floating_ip_pool=floating_ip_pool_id,
neutron_net_id=neutron_net_id,
volumes_per_node=self.config.get("volumes_per_node"),
volumes_size=self.config.get("volumes_size", 1),
node_configs=self.config.get("node_configs"),
cluster_configs=self.config.get("cluster_configs"),
wait_active=False)
self.context["sahara_clusters"][tenant_id] = cluster.id
# Need to save the client instance to poll for active status
wait_dict[cluster.id] = clients.sahara()
def all_active(dct):
for cl_id, client in dct.items():
cl = client.clusters.get(cl_id)
if cl.status.lower() != "active":
return False
return True
bench_utils.wait_for(
resource=wait_dict,
is_ready=all_active,
timeout=CONF.benchmark.cluster_create_timeout,
check_interval=CONF.benchmark.cluster_check_interval)
