def test_get_instances(self):
cluster = self._make_sample()
ctx = context.ctx()
idx = 0
ids = []
for ng in cluster.node_groups:
for i in range(ng.count):
idx += 1
ids.append(self.api.instance_add(ctx, ng, {
'instance_id': str(idx),
'instance_name': str(idx),
}))
cluster = self.api.cluster_get(ctx, cluster)
instances = cluster_utils.get_instances(cluster, ids)
ids = set()
for inst in instances:
ids.add(inst.instance_id)
self.assertEqual(idx, len(ids))
for i in range(1, idx):
self.assertIn(str(i), ids)
instances = cluster_utils.get_instances(cluster)
ids = set()
for inst in instances:
ids.add(inst.instance_id)
self.assertEqual(idx, len(ids))
for i in range(1, idx):
self.assertIn(str(i), ids)
def _launch_instances(self,
cluster,
target_count,
stages,
update_stack=False,
disable_rollback=True):
# create all instances
cluster = c_u.change_cluster_status(cluster, stages[0])
inst_ids = self._create_instances(cluster, target_count, update_stack,
disable_rollback)
# wait for all instances are up and networks ready
cluster = c_u.change_cluster_status(cluster, stages[1])
instances = c_u.get_instances(cluster, inst_ids)
self._await_networks(cluster, instances)
# prepare all instances
cluster = c_u.change_cluster_status(cluster, stages[2])
instances = c_u.get_instances(cluster, inst_ids)
volumes.mount_to_instances(instances)
self._configure_instances(cluster)
return inst_ids
def _launch_instances(self, cluster, target_count, stages,
update_stack=False, disable_rollback=True,
instances_to_delete=None):
# create all instances
cluster = c_u.change_cluster_status(cluster, stages[0])
inst_ids = self._create_instances(
cluster, target_count, update_stack, disable_rollback,
instances_to_delete)
# wait for all instances are up and networks ready
cluster = c_u.change_cluster_status(cluster, stages[1])
instances = c_u.get_instances(cluster, inst_ids)
self._await_networks(cluster, instances)
# prepare all instances
cluster = c_u.change_cluster_status(cluster, stages[2])
instances = c_u.get_instances(cluster, inst_ids)
volumes.mount_to_instances(instances)
self._configure_instances(cluster)
return inst_ids
def test_get_instances(self):
cluster = self._make_sample()
ctx = context.ctx()
idx = 0
ids = []
for ng in cluster.node_groups:
for i in range(ng.count):
idx += 1
ids.append(
self.api.instance_add(ctx, ng, {
'instance_id': str(idx),
'instance_name': str(idx),
}))
cluster = self.api.cluster_get(ctx, cluster)
instances = cluster_utils.get_instances(cluster, ids)
ids = set()
for inst in instances:
ids.add(inst.instance_id)
self.assertEqual(idx, len(ids))
for i in range(1, idx):
self.assertIn(str(i), ids)
instances = cluster_utils.get_instances(cluster)
ids = set()
for inst in instances:
ids.add(inst.instance_id)
self.assertEqual(idx, len(ids))
for i in range(1, idx):
self.assertIn(str(i), ids)
def create_cluster(self, cluster):
_warning_logger()
ctx = context.ctx()
self._update_rollback_strategy(cluster, shutdown=True)
# create all instances
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_SPAWNING)
self._create_instances(cluster)
# wait for all instances are up and networks ready
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_WAITING)
instances = c_u.get_instances(cluster)
self._await_active(cluster, instances)
self._assign_floating_ips(instances)
self._await_networks(cluster, instances)
cluster = conductor.cluster_get(ctx, cluster)
# attach volumes
volumes.attach_to_instances(c_u.get_instances(cluster))
# prepare all instances
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_PREPARING)
self._configure_instances(cluster)
self._update_rollback_strategy(cluster)
def _await_networks(self, cluster, instances):
if not instances:
return
cpo.add_provisioning_step(cluster.id, _("Assign IPs"), len(instances))
ips_assigned = set()
self._ips_assign(ips_assigned, cluster, instances)
LOG.info(
_LI("All instances have IPs assigned"))
cluster = conductor.cluster_get(context.ctx(), cluster)
instances = cluster_utils.get_instances(cluster, ips_assigned)
cpo.add_provisioning_step(
cluster.id, _("Wait for instance accessibility"), len(instances))
with context.ThreadGroup() as tg:
for instance in instances:
with context.set_current_instance_id(instance.instance_id):
tg.spawn("wait-for-ssh-%s" % instance.instance_name,
self._wait_until_accessible, instance)
LOG.info(_LI("All instances are accessible"))
def start_cluster(self, cluster):
keypairs.provision_keypairs(cluster)
des = cluster.description
backup = None
if len(des) > 5 and des[-6:] == "backup":
backup = "backup"
# 配置环境变量
s_scripts.config_user_env(cluster)
s_scripts.start_namenode(cluster, backup)
s_scripts.start_secondarynamenode(cluster)
s_scripts.start_resourcemanager(cluster)
run.start_dn_nm_processes(utils.get_instances(cluster))
run.await_datanodes(cluster)
s_scripts.start_historyserver(cluster)
s_scripts.start_oozie(self.pctx, cluster, backup)
s_scripts.start_hiveserver(self.pctx, cluster, backup)
# swift_helper.install_ssl_certs(cluster_utils.get_instances(cluster))
# start spark nodes, newly added 'swift_helper.install_ssl_certs' after 'start_spark'
s_scripts.start_spark(cluster)
swift_helper.install_ssl_certs(cluster_utils.get_instances(cluster))
# 新增的方法，配置用户的环境变量
# s_scripts.config_user_env(cluster)
# newly added
LOG.info('Cluster has been started successfully --by HXH')
self._set_cluster_info(cluster)
def test_attach(self, add_step, add_event, p_create_attach_vol, p_await, p_mount):
p_create_attach_vol.side_effect = ["/dev/vdb", "/dev/vdc"] * 2
p_await.return_value = None
p_mount.return_value = None
add_event.return_value = None
add_step.return_value = None
instance1 = {"id": "1", "instance_id": "123", "instance_name": "inst_1"}
instance2 = {"id": "2", "instance_id": "456", "instance_name": "inst_2"}
ng = {
"volumes_per_node": 2,
"volumes_size": 2,
"volumes_availability_zone": None,
"volume_mount_prefix": "/mnt/vols",
"volume_type": None,
"name": "master",
"cluster_id": "11",
"instances": [instance1, instance2],
"volume_local_to_instance": False,
}
cluster = r.ClusterResource({"node_groups": [ng]})
volumes.attach_to_instances(cluster_utils.get_instances(cluster))
self.assertEqual(4, p_create_attach_vol.call_count)
self.assertEqual(2, p_await.call_count)
self.assertEqual(1, p_mount.call_count)
def _provision_scaled_cluster(cluster_id, node_group_id_map):
ctx, cluster, plugin = _prepare_provisioning(cluster_id)
# Decommissioning surplus nodes with the plugin
cluster = c_u.change_cluster_status(cluster,
c_u.CLUSTER_STATUS_DECOMMISSIONING)
instances_to_delete = []
for node_group in cluster.node_groups:
new_count = node_group_id_map[node_group.id]
if new_count < node_group.count:
instances_to_delete += node_group.instances[new_count:node_group.
count]
if instances_to_delete:
context.set_step_type(_("Plugin: decommission cluster"))
plugin.decommission_nodes(cluster, instances_to_delete)
# Scaling infrastructure
cluster = c_u.change_cluster_status(cluster, c_u.CLUSTER_STATUS_SCALING)
context.set_step_type(_("Engine: scale cluster"))
instance_ids = INFRA.scale_cluster(cluster, node_group_id_map)
# Setting up new nodes with the plugin
if instance_ids:
ntp_service.configure_ntp(cluster_id)
cluster = c_u.change_cluster_status(cluster,
c_u.CLUSTER_STATUS_CONFIGURING)
instances = c_u.get_instances(cluster, instance_ids)
context.set_step_type(_("Plugin: scale cluster"))
plugin.scale_cluster(cluster, instances)
c_u.change_cluster_status(cluster, c_u.CLUSTER_STATUS_ACTIVE)
_refresh_health_for_cluster(cluster_id)
def _provision_scaled_cluster(cluster_id, node_group_id_map):
ctx, cluster, plugin = _prepare_provisioning(cluster_id)
# Decommissioning surplus nodes with the plugin
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_DECOMMISSIONING)
instances_to_delete = []
for node_group in cluster.node_groups:
new_count = node_group_id_map[node_group.id]
if new_count < node_group.count:
instances_to_delete += node_group.instances[new_count:
node_group.count]
if instances_to_delete:
context.set_step_type(_("Plugin: decommission cluster"))
plugin.decommission_nodes(cluster, instances_to_delete)
# Scaling infrastructure
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_SCALING)
context.set_step_type(_("Engine: scale cluster"))
instance_ids = INFRA.scale_cluster(cluster, node_group_id_map)
# Setting up new nodes with the plugin
if instance_ids:
ntp_service.configure_ntp(cluster_id)
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_CONFIGURING)
instances = c_u.get_instances(cluster, instance_ids)
context.set_step_type(_("Plugin: scale cluster"))
plugin.scale_cluster(cluster, instances)
c_u.change_cluster_status(cluster, c_u.CLUSTER_STATUS_ACTIVE)
def _await_networks(self, cluster, instances):
if not instances:
return
cpo.add_provisioning_step(cluster.id, _("Assign IPs"), len(instances))
ips_assigned = set()
self._ips_assign(ips_assigned, cluster, instances)
LOG.info("All instances have IPs assigned")
cluster = conductor.cluster_get(context.ctx(), cluster)
instances = cluster_utils.get_instances(cluster, ips_assigned)
cpo.add_provisioning_step(cluster.id,
_("Wait for instance accessibility"),
len(instances))
with context.ThreadGroup() as tg:
for instance in instances:
with context.set_current_instance_id(instance.instance_id):
tg.spawn("wait-for-ssh-%s" % instance.instance_name,
self._wait_until_accessible, instance)
LOG.info("All instances are accessible")
def test_attach(self, add_step, add_event, p_create_attach_vol, p_await,
p_mount):
p_create_attach_vol.side_effect = ['/dev/vdb', '/dev/vdc'] * 2
p_await.return_value = None
p_mount.return_value = None
add_event.return_value = None
add_step.return_value = None
instance1 = {'id': '1',
'instance_id': '123',
'instance_name': 'inst_1'}
instance2 = {'id': '2',
'instance_id': '456',
'instance_name': 'inst_2'}
ng = {'volumes_per_node': 2,
'volumes_size': 2,
'volumes_availability_zone': None,
'volume_mount_prefix': '/mnt/vols',
'volume_type': None,
'name': 'master',
'cluster_id': '11',
'instances': [instance1, instance2],
'volume_local_to_instance': False}
cluster = r.ClusterResource({'node_groups': [ng]})
volumes.attach_to_instances(cluster_utils.get_instances(cluster))
self.assertEqual(4, p_create_attach_vol.call_count)
self.assertEqual(2, p_await.call_count)
self.assertEqual(1, p_mount.call_count)
def provision_keypairs(cluster, instances=None):
extra = cluster.extra.to_dict() if cluster.extra else {}
# use same keypair for scaling
keypair = extra.get('vanilla_keypair')
if not instances:
instances = utils.get_instances(cluster)
else:
# scaling
if not keypair:
# cluster created before mitaka, skipping provisioning
return
if not keypair:
private, public = crypto.generate_key_pair()
keypair = {'public': public, 'private': private}
extra['vanilla_keypair'] = keypair
extra['vanilla_keypair']['private'] = _store_secret(
keypair['private'])
cond.cluster_update(context.ctx(), cluster, {'extra': extra})
else:
keypair['private'] = _get_secret(keypair['private'])
with context.ThreadGroup() as tg:
for instance in instances:
tg.spawn(
'provision-key-%s' % instance.instance_name,
_provision_key, instance, keypair)
def configure_ntp(cluster_id):
cluster = conductor.cluster_get(context.ctx(), cluster_id)
if not is_ntp_enabled(cluster):
LOG.debug("Don't configure NTP on cluster")
return
instances = c_u.get_instances(cluster)
url = retrieve_ntp_server_url(cluster)
with context.ThreadGroup() as tg:
for instance in instances:
tg.spawn("configure-ntp-%s" % instance.instance_name, _configure_ntp_on_instance, instance, url)
def check_health(self):
instances = cluster_utils.get_instances(self.cluster)
try:
for inst in instances:
self._check_health_for_instance(inst)
except Exception:
LOG.exception(_LE(
"Some instances in the cluster are not available"))
raise RedHealthError(_("Some instances are not available"))
return _("All instances are available")
def configure_ntp(cluster_id):
cluster = conductor.cluster_get(context.ctx(), cluster_id)
if not is_ntp_enabled(cluster):
LOG.debug("Don't configure NTP on cluster")
return
instances = c_u.get_instances(cluster)
url = retrieve_ntp_server_url(cluster)
with context.ThreadGroup() as tg:
for instance in instances:
tg.spawn("configure-ntp-%s" % instance.instance_name,
_configure_ntp_on_instance, instance, url)
def get_alerts_data(self, check_type=None):
if check_type and self._data is not None:
# return cached data
return self._data.get(check_type, [])
self._data = {}
self._instances = cluster_utils.get_instances(self._cluster)
lock = threading.Lock()
with context.ThreadGroup() as tg:
for ins in self._instances:
tg.spawn('Get health check data of instance %s' % ins.id,
self._instance_get_data, ins, lock)
return self._data
def get_alerts_data(self, check_type=None):
if check_type and self._data is not None:
# return cached data
return self._data.get(check_type, [])
self._data = {}
self._instances = cluster_utils.get_instances(self._cluster)
lock = threading.Lock()
with context.ThreadGroup() as tg:
for ins in self._instances:
tg.spawn('Get health check data of instance %s' % ins.id,
self._instance_get_data, ins, lock)
return self._data
def setup_clients(cluster, server=None, instances=None):
if not instances:
instances = cl_utils.get_instances(cluster)
server_ip = None
cpo.add_provisioning_step(cluster.id, _("Setting Up Kerberos clients"),
len(instances))
if not server:
server_ip = get_kdc_server_ip(cluster)
with context.ThreadGroup() as tg:
for instance in instances:
tg.spawn('setup-client-%s' % instance.instance_name,
_setup_client_node, cluster, instance, server, server_ip)
def _provision_scaled_cluster(cluster_id,
node_group_id_map,
node_group_instance_map=None):
ctx, cluster, plugin = _prepare_provisioning(cluster_id)
# Decommissioning surplus nodes with the plugin
cluster = c_u.change_cluster_status(cluster,
c_u.CLUSTER_STATUS_DECOMMISSIONING)
try:
instances_to_delete = []
for node_group in cluster.node_groups:
new_count = node_group_id_map[node_group.id]
if new_count < node_group.count:
if (node_group_instance_map
and node_group.id in node_group_instance_map):
for instance_ref in node_group_instance_map[node_group.id]:
instance = _get_instance_obj(node_group.instances,
instance_ref)
instances_to_delete.append(instance)
while node_group.count - new_count > len(instances_to_delete):
instances_to_delete.append(
_get_random_instance_from_ng(node_group.instances,
instances_to_delete))
if instances_to_delete:
context.set_step_type(_("Plugin: decommission cluster"))
plugin.decommission_nodes(cluster, instances_to_delete)
# Scaling infrastructure
cluster = c_u.change_cluster_status(cluster,
c_u.CLUSTER_STATUS_SCALING)
context.set_step_type(_("Engine: scale cluster"))
instance_ids = INFRA.scale_cluster(cluster, node_group_id_map,
instances_to_delete)
# Setting up new nodes with the plugin
if instance_ids:
ntp_service.configure_ntp(cluster_id, instance_ids)
cluster = c_u.change_cluster_status(cluster,
c_u.CLUSTER_STATUS_CONFIGURING)
instances = c_u.get_instances(cluster, instance_ids)
context.set_step_type(_("Plugin: scale cluster"))
plugin.scale_cluster(cluster, instances)
c_u.change_cluster_status(cluster, c_u.CLUSTER_STATUS_ACTIVE)
_refresh_health_for_cluster(cluster_id)
except Exception as e:
c_u.change_cluster_status(cluster, c_u.CLUSTER_STATUS_ACTIVE,
six.text_type(e))
def setup_clients(cluster, server=None, instances=None):
if not instances:
instances = cl_utils.get_instances(cluster)
server_ip = None
cpo.add_provisioning_step(
cluster.id, _("Setting Up Kerberos clients"), len(instances))
if not server:
server_ip = get_kdc_server_ip(cluster)
with context.ThreadGroup() as tg:
for instance in instances:
tg.spawn('setup-client-%s' % instance.instance_name,
_setup_client_node, cluster, instance,
server, server_ip)
def _get_instance_if_running(self, job_execution):
pid, inst_id = self._get_pid_and_inst_id(job_execution.oozie_job_id)
if not pid or not inst_id or (
job_execution.info['status'] in edp.JOB_STATUSES_TERMINATED):
return None, None
# TODO(tmckay): well, if there is a list index out of range
# error here it probably means that the instance is gone. If we
# have a job execution that is not terminated, and the instance
# is gone, we should probably change the status somehow.
# For now, do nothing.
try:
instance = c_u.get_instances(self.cluster, [inst_id])[0]
except Exception:
instance = None
return pid, instance
def start_cluster(self, cluster):
s_scripts.start_namenode(cluster)
s_scripts.start_secondarynamenode(cluster)
s_scripts.start_resourcemanager(cluster)
run.start_dn_nm_processes(utils.get_instances(cluster))
run.await_datanodes(cluster)
s_scripts.start_historyserver(cluster)
s_scripts.start_oozie(self.pctx, cluster)
s_scripts.start_hiveserver(self.pctx, cluster)
swift_helper.install_ssl_certs(cluster_utils.get_instances(cluster))
self._set_cluster_info(cluster)
def scale_cluster(self, cluster, node_group_id_map):
_warning_logger()
ctx = context.ctx()
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_SCALING_SPAWNING)
instance_ids = self._scale_cluster_instances(cluster,
node_group_id_map)
self._update_rollback_strategy(cluster, instance_ids=instance_ids)
cluster = conductor.cluster_get(ctx, cluster)
c_u.clean_cluster_from_empty_ng(cluster)
cluster = conductor.cluster_get(ctx, cluster)
instances = c_u.get_instances(cluster, instance_ids)
self._await_active(cluster, instances)
self._assign_floating_ips(instances)
self._await_networks(cluster, instances)
cluster = conductor.cluster_get(ctx, cluster)
volumes.attach_to_instances(
c_u.get_instances(cluster, instance_ids))
# we should be here with valid cluster: if instances creation
# was not successful all extra-instances will be removed above
if instance_ids:
self._configure_instances(cluster)
self._update_rollback_strategy(cluster)
return instance_ids
def _get_instance_if_running(self, job_execution):
pid, inst_id = self._get_pid_and_inst_id(job_execution.engine_job_id)
if not pid or not inst_id or (
job_execution.info['status'] in edp.JOB_STATUSES_TERMINATED):
return None, None
# TODO(tmckay): well, if there is a list index out of range
# error here it probably means that the instance is gone. If we
# have a job execution that is not terminated, and the instance
# is gone, we should probably change the status somehow.
# For now, do nothing.
try:
instance = c_u.get_instances(self.cluster, [inst_id])[0]
except Exception:
instance = None
return pid, instance
def start_cluster(self, cluster):
s_scripts.start_namenode(cluster)
s_scripts.start_secondarynamenode(cluster)
s_scripts.start_resourcemanager(cluster)
run.start_dn_nm_processes(utils.get_instances(cluster))
run.await_datanodes(cluster)
s_scripts.start_historyserver(cluster)
s_scripts.start_oozie(self.pctx, cluster)
s_scripts.start_hiveserver(self.pctx, cluster)
swift_helper.install_ssl_certs(cluster_utils.get_instances(cluster))
self._set_cluster_info(cluster)
def _populate_cluster(self, cluster, stack):
ctx = context.ctx()
old_ids = [i.instance_id for i in c_u.get_instances(cluster)]
new_ids = []
for node_group in cluster.node_groups:
instances = stack.get_node_group_instances(node_group)
for instance in instances:
nova_id = instance['physical_id']
name = instance['name']
if nova_id not in old_ids:
instance_id = conductor.instance_add(
ctx, node_group, {"instance_id": nova_id,
"instance_name": name})
new_ids.append(instance_id)
return new_ids
def _populate_cluster(self, cluster, stack):
ctx = context.ctx()
old_ids = [i.instance_id for i in c_u.get_instances(cluster)]
new_ids = []
for node_group in cluster.node_groups:
instances = stack.get_node_group_instances(node_group)
for instance in instances:
nova_id = instance['physical_id']
name = instance['name']
if nova_id not in old_ids:
instance_id = conductor.instance_add(
ctx, node_group, {
"instance_id": nova_id,
"instance_name": name
})
new_ids.append(instance_id)
return new_ids
def test_delete_floating_ips(self, map_mock, deleted_checker):
node_groups = [_make_ng_dict("test_group_1", "test_flavor",
["data node", "test tracker"], 2, 'pool')]
map_mock.return_value = []
ctx = context.ctx()
cluster = _create_cluster_mock(node_groups, ["datanode"])
self.engine._create_instances(cluster)
cluster = conductor.cluster_get(ctx, cluster)
instances_list = cluster_utils.get_instances(cluster)
self.engine._assign_floating_ips(instances_list)
deleted_checker.return_value = True
self.engine._shutdown_instances(cluster)
self.assertEqual(2, self.nova.floating_ips.delete.call_count,
"Not expected floating IPs number found in delete")
self.assertEqual(2, self.nova.servers.delete.call_count,
"Not expected")
def _populate_cluster(self, cluster, stack):
ctx = context.ctx()
old_ids = [i.instance_id for i in c_u.get_instances(cluster)]
new_ids = []
for node_group in cluster.node_groups:
instances = stack.get_node_group_instances(node_group)
for instance in instances:
nova_id = instance['physical_id']
if nova_id not in old_ids:
name = instance['name']
inst = {"instance_id": nova_id, "instance_name": name}
if cluster.use_designate_feature():
inst.update({
"dns_hostname":
name + '.' + cluster.domain_name[:-1]
})
instance_id = conductor.instance_add(ctx, node_group, inst)
new_ids.append(instance_id)
return new_ids
def test_delete_floating_ips(self, map_mock, deleted_checker):
node_groups = [
_make_ng_dict("test_group_1", "test_flavor",
["data node", "test tracker"], 2, 'pool')
]
map_mock.return_value = []
ctx = context.ctx()
cluster = _create_cluster_mock(node_groups, ["datanode"])
self.engine._create_instances(cluster)
cluster = conductor.cluster_get(ctx, cluster)
instances_list = cluster_utils.get_instances(cluster)
self.engine._assign_floating_ips(instances_list)
deleted_checker.return_value = True
self.engine._shutdown_instances(cluster)
self.assertEqual(2, self.nova.floating_ips.delete.call_count,
"Not expected floating IPs number found in delete")
self.assertEqual(2, self.nova.servers.delete.call_count,
"Not expected")
def rollback_cluster(self, cluster, reason):
_warning_logger()
rollback_info = cluster.rollback_info or {}
self._update_rollback_strategy(cluster)
if rollback_info.get('shutdown', False):
self._rollback_cluster_creation(cluster, reason)
LOG.warning(_LW("Cluster creation rollback "
"(reason: {reason})").format(reason=reason))
return False
instance_ids = rollback_info.get('instance_ids', [])
if instance_ids:
self._rollback_cluster_scaling(
cluster,
c_u.get_instances(cluster, instance_ids), reason)
LOG.warning(_LW("Cluster scaling rollback "
"(reason: {reason})").format(reason=reason))
return True
return False
def test_attach(self, add_step, add_event, p_create_attach_vol, p_await,
p_mount):
p_create_attach_vol.side_effect = ['/dev/vdb', '/dev/vdc'] * 2
p_await.return_value = None
p_mount.return_value = None
add_event.return_value = None
add_step.return_value = None
instance1 = {
'id': '1',
'instance_id': '123',
'instance_name': 'inst_1'
}
instance2 = {
'id': '2',
'instance_id': '456',
'instance_name': 'inst_2'
}
ng = {
'volumes_per_node': 2,
'volumes_size': 2,
'volumes_availability_zone': None,
'volume_mount_prefix': '/mnt/vols',
'volume_type': None,
'name': 'master',
'cluster_id': '11',
'instances': [instance1, instance2],
'volume_local_to_instance': False
}
cluster = r.ClusterResource({'node_groups': [ng]})
volumes.attach_to_instances(cluster_utils.get_instances(cluster))
self.assertEqual(4, p_create_attach_vol.call_count)
self.assertEqual(2, p_await.call_count)
self.assertEqual(1, p_mount.call_count)
def test_ip_assignment_use_no_floating(self):
self.override_config("use_floating_ips", False)
node_groups = [_make_ng_dict("test_group_1", "test_flavor",
["data node", "test tracker"], 2,
'pool'),
_make_ng_dict("test_group_2", "test_flavor",
["name node", "test tracker"], 1)]
ctx = context.ctx()
cluster = _create_cluster_mock(node_groups, ["data node"])
self.engine._create_instances(cluster)
cluster = conductor.cluster_get(ctx, cluster)
instances_list = cluster_utils.get_instances(cluster)
self.engine._assign_floating_ips(instances_list)
self.nova.floating_ips.create.assert_has_calls(
[mock.call("pool"), mock.call("pool")])
self.assertEqual(2, self.nova.floating_ips.create.call_count,
"Not expected floating IPs number found.")
def provision_keypairs(cluster, instances=None):
extra = cluster.extra.to_dict() if cluster.extra else {}
# use same keypair for scaling
keypair = extra.get('vanilla_keypair')
if not instances:
instances = utils.get_instances(cluster)
else:
# scaling
if not keypair:
# cluster created before mitaka, skipping provisioning
return
if not keypair:
private, public = crypto.generate_key_pair()
keypair = {'public': public, 'private': private}
extra['vanilla_keypair'] = keypair
extra['vanilla_keypair']['private'] = _store_secret(keypair['private'])
cond.cluster_update(context.ctx(), cluster, {'extra': extra})
else:
keypair['private'] = _get_secret(keypair['private'])
with context.ThreadGroup() as tg:
for instance in instances:
tg.spawn('provision-key-%s' % instance.instance_name,
_provision_key, instance, keypair)
def _populate_cluster(self, cluster, stack):
ctx = context.ctx()
old_ids = [i.instance_id for i in c_u.get_instances(cluster)]
new_ids = []
for node_group in cluster.node_groups:
instances = stack.get_node_group_instances(node_group)
for instance in instances:
nova_id = instance['physical_id']
if nova_id not in old_ids:
name = instance['name']
inst = {
"instance_id": nova_id,
"instance_name": name
}
if cluster.use_designate_feature():
inst.update(
{"dns_hostname":
name + '.' + cluster.domain_name[:-1]})
instance_id = conductor.instance_add(ctx, node_group, inst)
new_ids.append(instance_id)
return new_ids
def test_ip_assignment_use_no_floating(self):
self.override_config("use_floating_ips", False)
node_groups = [
_make_ng_dict("test_group_1", "test_flavor",
["data node", "test tracker"], 2, 'pool'),
_make_ng_dict("test_group_2", "test_flavor",
["name node", "test tracker"], 1)
]
ctx = context.ctx()
cluster = _create_cluster_mock(node_groups, ["data node"])
self.engine._create_instances(cluster)
cluster = conductor.cluster_get(ctx, cluster)
instances_list = cluster_utils.get_instances(cluster)
self.engine._assign_floating_ips(instances_list)
self.nova.floating_ips.create.assert_has_calls(
[mock.call("pool"), mock.call("pool")])
self.assertEqual(2, self.nova.floating_ips.create.call_count,
"Not expected floating IPs number found.")
def _provision_scaled_cluster(cluster_id, node_group_id_map,
node_group_instance_map=None):
"""Provision scaled cluster.
:param cluster_id: Id of cluster to be scaled.
:param node_group_id_map: Dictionary in the format
node_group_id: number of instances.
:param node_group_instance_map: Specifies the instances to be removed in
each node group.
"""
ctx, cluster, plugin = _prepare_provisioning(cluster_id)
# Decommissioning surplus nodes with the plugin
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_DECOMMISSIONING)
try:
instances_to_delete = []
for node_group in cluster.node_groups:
ng_inst_to_delete_count = 0
# new_count is the new number of instance on the current node group
new_count = node_group_id_map[node_group.id]
if new_count < node_group.count:
# Adding selected instances to delete to the list
if _specific_inst_to_delete(node_group,
node_group_instance_map):
for instance_ref in node_group_instance_map[node_group.id]:
instances_to_delete.append(_get_instance_obj(
node_group.instances, instance_ref))
ng_inst_to_delete_count += 1
# Adding random instances to the list when the number of
# specific instances does not equals the difference between the
# current count and the new count of instances.
while node_group.count - new_count > ng_inst_to_delete_count:
instances_to_delete.append(_get_random_instance_from_ng(
node_group.instances, instances_to_delete))
ng_inst_to_delete_count += 1
if instances_to_delete:
context.set_step_type(_("Plugin: decommission cluster"))
plugin.decommission_nodes(cluster, instances_to_delete)
# Scaling infrastructure
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_SCALING)
context.set_step_type(_("Engine: scale cluster"))
instance_ids = INFRA.scale_cluster(cluster, node_group_id_map,
instances_to_delete)
# Setting up new nodes with the plugin
if instance_ids:
ntp_service.configure_ntp(cluster_id, instance_ids)
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_CONFIGURING)
instances = c_u.get_instances(cluster, instance_ids)
context.set_step_type(_("Plugin: scale cluster"))
plugin.scale_cluster(cluster, instances)
c_u.change_cluster_status(cluster, c_u.CLUSTER_STATUS_ACTIVE)
_refresh_health_for_cluster(cluster_id)
except Exception as e:
c_u.change_cluster_status(cluster, c_u.CLUSTER_STATUS_ACTIVE,
six.text_type(e))
def _remove_db_objects(self, cluster):
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster)
instances = cluster_utils.get_instances(cluster)
for inst in instances:
conductor.instance_remove(ctx, inst)
def _remove_db_objects(self, cluster):
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster)
instances = cluster_utils.get_instances(cluster)
for inst in instances:
conductor.instance_remove(ctx, inst)
def cluster_get_instances(cluster, instances_ids=None, **kwargs):
return cluster_utils.get_instances(cluster, instances_ids)
