def _validate_existing_ng_scaling(self, cluster, existing): scalable_processes = self._get_scalable_processes() dn_to_delete = 0 for ng in cluster.node_groups: if ng.id in existing: if ng.count > existing[ng.id] and ("datanode" in ng.node_processes): dn_to_delete += ng.count - existing[ng.id] if not set(ng.node_processes).issubset(scalable_processes): raise ex.NodeGroupCannotBeScaled( ng.name, _("Spark plugin cannot scale nodegroup" " with processes: %s") % ' '.join(ng.node_processes)) dn_amount = len(utils.get_instances(cluster, "datanode")) rep_factor = utils.get_config_value_or_default('HDFS', "dfs.replication", cluster) if dn_to_delete > 0 and dn_amount - dn_to_delete < rep_factor: raise ex.ClusterCannotBeScaled( cluster.name, _("Spark plugin cannot shrink cluster because " "there would be not enough nodes for HDFS " "replicas (replication factor is %s)") % rep_factor)
def get_cluster_spec(self, cluster, user_inputs, scaled_groups=None, cluster_template=None): if cluster_template: cluster_spec = cs.ClusterSpec(cluster_template) else: if scaled_groups: for ng in cluster.node_groups: ng_id = ng['id'] if (ng_id in scaled_groups and ng['count'] > scaled_groups[ng_id]): raise ex.ClusterCannotBeScaled( cluster.name, _('The HDP plugin does not support ' 'the decommissioning of nodes ' 'for HDP version 1.3.2')) cluster_spec = self.get_default_cluster_configuration() cluster_spec.create_operational_config(cluster, user_inputs, scaled_groups) cs.validate_number_of_datanodes(cluster, scaled_groups, self.get_config_items()) return cluster_spec
def validate_zookeeper_node_count(zk_ng, existing, additional): zk_amount = 0 for ng in zk_ng: if ng.id in existing: zk_amount += existing[ng.id] else: zk_amount += ng.count for ng_id in additional: ng = u.get_by_id(zk_ng, ng_id) if "zookeeper" in ng.node_processes: zk_amount += ng.count if (zk_amount % 2) != 1: msg = _("Vanilla plugin cannot scale cluster because it must keep" " zookeeper service in odd.") raise ex.ClusterCannotBeScaled(zk_ng[0].cluster.name, msg)
def validate_existing_ng_scaling(cluster, existing): scalable_processes = _get_scalable_processes() dn_to_delete = 0 for ng in cluster.node_groups: if ng.id in existing: if (ng.count > existing[ng.id] and 'HDFS_DATANODE' in ng.node_processes): dn_to_delete += ng.count - existing[ng.id] if not set(ng.node_processes).issubset(scalable_processes): msg = _("CDH plugin cannot scale nodegroup with processes: " "%(processes)s") raise ex.NodeGroupCannotBeScaled( ng.name, msg % {'processes': ' '.join(ng.node_processes)}) dn_count = _get_inst_count(cluster, 'HDFS_DATANODE') - dn_to_delete replicas = PU.get_config_value('HDFS', 'dfs_replication', cluster) if dn_count < replicas: raise ex.ClusterCannotBeScaled( cluster, _('Number of datanodes must be not less than dfs_replication.'))
def validate_existing_ng_scaling(pctx, cluster, existing): scalable_processes = _get_scalable_processes() dn_to_delete = 0 for ng in cluster.node_groups: if ng.id in existing: if ng.count > existing[ng.id] and "datanode" in ng.node_processes: dn_to_delete += ng.count - existing[ng.id] if not set(ng.node_processes).issubset(scalable_processes): msg = _("Vanilla plugin cannot scale nodegroup " "with processes: %s") raise ex.NodeGroupCannotBeScaled( ng.name, msg % ' '.join(ng.node_processes)) dn_amount = len(vu.get_datanodes(cluster)) rep_factor = cu.get_config_value(pctx, 'HDFS', 'dfs.replication', cluster) if dn_to_delete > 0 and dn_amount - dn_to_delete < rep_factor: msg = _("Vanilla plugin cannot shrink cluster because it would be " "not enough nodes for replicas (replication factor is %s)") raise ex.ClusterCannotBeScaled(cluster.name, msg % rep_factor)