def validate(self, cluster):
if cluster.hadoop_version == "1.0.0":
raise exceptions.DeprecatedException(
_("Support for Spark version 1.0.0 is now deprecated and will" " be removed in the 2016.1 release.")
)
nn_count = sum([ng.count for ng in utils.get_node_groups(cluster, "namenode")])
if nn_count != 1:
raise ex.InvalidComponentCountException("namenode", 1, nn_count)
dn_count = sum([ng.count for ng in utils.get_node_groups(cluster, "datanode")])
if dn_count < 1:
raise ex.InvalidComponentCountException("datanode", _("1 or more"), nn_count)
rep_factor = utils.get_config_value_or_default("HDFS", "dfs.replication", cluster)
if dn_count < rep_factor:
raise ex.InvalidComponentCountException(
"datanode",
_("%s or more") % rep_factor,
dn_count,
_("Number of %(dn)s instances should not be less " "than %(replication)s")
% {"dn": "datanode", "replication": "dfs.replication"},
)
# validate Spark Master Node and Spark Slaves
sm_count = sum([ng.count for ng in utils.get_node_groups(cluster, "master")])
if sm_count != 1:
raise ex.RequiredServiceMissingException("Spark master")
sl_count = sum([ng.count for ng in utils.get_node_groups(cluster, "slave")])
if sl_count < 1:
raise ex.InvalidComponentCountException("Spark slave", _("1 or more"), sl_count)
def validate(self, cluster):
nn_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "namenode")])
if nn_count != 1:
raise ex.InvalidComponentCountException("namenode", 1, nn_count)
dn_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "datanode")])
if dn_count < 1:
raise ex.InvalidComponentCountException("datanode", _("1 or more"),
nn_count)
# validate Spark Master Node and Spark Slaves
sm_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "master")])
if sm_count != 1:
raise ex.RequiredServiceMissingException("Spark master")
sl_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "slave")])
if sl_count < 1:
raise ex.InvalidComponentCountException("Spark slave",
_("1 or more"),
sl_count)
def _impala_validation(cls, cluster):
ics_count = cls.get_inst_count(cluster, 'IMPALA_CATALOGSERVER')
iss_count = cls.get_inst_count(cluster, 'IMPALA_STATESTORE')
id_count = cls.get_inst_count(cluster, 'IMPALAD')
dn_count = cls.get_inst_count(cluster, 'HDFS_DATANODE')
hms_count = cls.get_inst_count(cluster, 'HIVE_METASTORE')
if ics_count > 1:
raise ex.InvalidComponentCountException('IMPALA_CATALOGSERVER',
_('0 or 1'), ics_count)
if iss_count > 1:
raise ex.InvalidComponentCountException('IMPALA_STATESTORE',
_('0 or 1'), iss_count)
if ics_count == 1:
datanode_ng = u.get_node_groups(cluster, "HDFS_DATANODE")
impalad_ng = u.get_node_groups(cluster, "IMPALAD")
datanodes = set(ng.id for ng in datanode_ng)
impalads = set(ng.id for ng in impalad_ng)
if datanodes != impalads:
raise ex.InvalidClusterTopology(
_("IMPALAD must be installed on every HDFS_DATANODE"))
if iss_count != 1:
raise ex.RequiredServiceMissingException('IMPALA_STATESTORE',
required_by='IMPALA')
if id_count < 1:
raise ex.RequiredServiceMissingException('IMPALAD',
required_by='IMPALA')
if dn_count < 1:
raise ex.RequiredServiceMissingException('HDFS_DATANODE',
required_by='IMPALA')
if hms_count < 1:
raise ex.RequiredServiceMissingException('HIVE_METASTORE',
required_by='IMPALA')
def _impala_validation(cls, cluster):
ics_count = cls._get_inst_count(cluster, 'IMPALA_CATALOGSERVER')
iss_count = cls._get_inst_count(cluster, 'IMPALA_STATESTORE')
id_count = cls._get_inst_count(cluster, 'IMPALAD')
dn_count = cls._get_inst_count(cluster, 'HDFS_DATANODE')
hms_count = cls._get_inst_count(cluster, 'HIVE_METASTORE')
if ics_count > 1:
raise ex.InvalidComponentCountException('IMPALA_CATALOGSERVER',
_('0 or 1'), ics_count)
if iss_count > 1:
raise ex.InvalidComponentCountException('IMPALA_STATESTORE',
_('0 or 1'), iss_count)
if ics_count == 1:
datanode_ng = u.get_node_groups(cluster, "HDFS_DATANODE")
impalad_ng = u.get_node_groups(cluster, "IMPALAD")
datanodes = set(ng.id for ng in datanode_ng)
impalads = set(ng.id for ng in impalad_ng)
if datanodes != impalads:
raise ex.InvalidClusterTopology(
_("IMPALAD must be installed on every HDFS_DATANODE"))
if iss_count != 1:
raise ex.RequiredServiceMissingException(
'IMPALA_STATESTORE', required_by='IMPALA')
if id_count < 1:
raise ex.RequiredServiceMissingException(
'IMPALAD', required_by='IMPALA')
if dn_count < 1:
raise ex.RequiredServiceMissingException(
'HDFS_DATANODE', required_by='IMPALA')
if hms_count < 1:
raise ex.RequiredServiceMissingException(
'HIVE_METASTORE', required_by='IMPALA')
def validate(self, cluster):
nn_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "namenode")])
if nn_count != 1:
raise ex.InvalidComponentCountException("namenode", 1, nn_count)
dn_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "datanode")])
if dn_count < 1:
raise ex.InvalidComponentCountException("datanode", _("1 or more"),
nn_count)
# validate Spark Master Node and Spark Slaves
sm_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "master")])
if sm_count != 1:
raise ex.RequiredServiceMissingException("Spark master")
sl_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "slave")])
if sl_count < 1:
raise ex.InvalidComponentCountException("Spark slave",
_("1 or more"), sl_count)
def validate(self, cluster):
nn_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "namenode")])
if nn_count != 1:
raise ex.InvalidComponentCountException("namenode", 1, nn_count)
dn_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "datanode")])
if dn_count < 1:
raise ex.InvalidComponentCountException("datanode", _("1 or more"),
nn_count)
rep_factor = c_helper.get_config_value('HDFS', "dfs.replication",
cluster)
if dn_count < rep_factor:
raise ex.InvalidComponentCountException(
'datanode', _('%s or more') % rep_factor, dn_count,
_('Number of %(dn)s instances should not be less '
'than %(replication)s')
% {'dn': 'datanode', 'replication': 'dfs.replication'})
# validate Spark Master Node and Spark Slaves
sm_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "master")])
if sm_count != 1:
raise ex.RequiredServiceMissingException("Spark master")
sl_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "slave")])
if sl_count < 1:
raise ex.InvalidComponentCountException("Spark slave",
_("1 or more"),
sl_count)
def validate(self, cluster):
# validate Storm Master Node and Storm Slaves
sm_count = sum([ng.count for ng in utils.get_node_groups(cluster, "nimbus")])
if sm_count != 1:
raise ex.RequiredServiceMissingException("Storm nimbus")
sl_count = sum([ng.count for ng in utils.get_node_groups(cluster, "supervisor")])
if sl_count < 1:
raise ex.InvalidComponentCountException("Storm supervisor", _("1 or more"), sl_count)
def test_get_node_groups(self):
res = pu.get_node_groups(self.cluster)
self.assertEqual([
FakeNodeGroup(["node_process1"]),
FakeNodeGroup(["node_process2"]),
FakeNodeGroup(["node_process3"]),
], res)
res = pu.get_node_groups(self.cluster, "node_process1")
self.assertEqual([FakeNodeGroup(["node_process1"])], res)
res = pu.get_node_groups(self.cluster, "node_process")
self.assertEqual([], res)
def validate(self, cluster):
# validate Storm Master Node and Storm Slaves
sm_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "nimbus")])
if sm_count != 1:
raise ex.RequiredServiceMissingException("Storm nimbus")
sl_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "supervisor")])
if sl_count < 1:
raise ex.InvalidComponentCountException("Storm supervisor",
_("1 or more"), sl_count)
def test_get_node_groups(self):
res = pu.get_node_groups(self.cluster)
self.assertEqual([
FakeNodeGroup(["node_process1"]),
FakeNodeGroup(["node_process2"]),
FakeNodeGroup(["node_process3"]),
], res)
res = pu.get_node_groups(self.cluster, "node_process1")
self.assertEqual([
FakeNodeGroup(["node_process1"])
], res)
res = pu.get_node_groups(self.cluster, "node_process")
self.assertEqual([], res)
def validate(self, cluster):
if cluster.hadoop_version == "1.0.0":
raise exceptions.DeprecatedException(
_("Support for Spark version 1.0.0 is now deprecated and will"
" be removed in the 2016.1 release."))
nn_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "namenode")])
if nn_count != 1:
raise ex.InvalidComponentCountException("namenode", 1, nn_count)
dn_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "datanode")])
if dn_count < 1:
raise ex.InvalidComponentCountException("datanode", _("1 or more"),
nn_count)
rep_factor = utils.get_config_value_or_default('HDFS',
"dfs.replication",
cluster)
if dn_count < rep_factor:
raise ex.InvalidComponentCountException(
'datanode',
_('%s or more') % rep_factor, dn_count,
_('Number of %(dn)s instances should not be less '
'than %(replication)s') % {
'dn': 'datanode',
'replication': 'dfs.replication'
})
# validate Spark Master Node and Spark Slaves
sm_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "master")])
if sm_count != 1:
raise ex.RequiredServiceMissingException("Spark master")
sl_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "slave")])
if sl_count < 1:
raise ex.InvalidComponentCountException("Spark slave",
_("1 or more"), sl_count)
def validate(self, cluster):
nn_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "namenode")])
if nn_count != 1:
raise ex.InvalidComponentCountException("namenode", 1, nn_count)
jt_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "jobtracker")])
if jt_count not in [0, 1]:
raise ex.InvalidComponentCountException("jobtracker", _('0 or 1'),
jt_count)
oozie_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "oozie")])
if oozie_count not in [0, 1]:
raise ex.InvalidComponentCountException("oozie", _('0 or 1'),
oozie_count)
hive_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "hiveserver")])
if jt_count == 0:
tt_count = sum([
ng.count
for ng in utils.get_node_groups(cluster, "tasktracker")
])
if tt_count > 0:
raise ex.RequiredServiceMissingException(
"jobtracker", required_by="tasktracker")
if oozie_count > 0:
raise ex.RequiredServiceMissingException("jobtracker",
required_by="oozie")
if hive_count > 0:
raise ex.RequiredServiceMissingException("jobtracker",
required_by="hive")
if hive_count not in [0, 1]:
raise ex.InvalidComponentCountException("hive", _('0 or 1'),
hive_count)
def validate(self, cluster):
nn_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "namenode")])
if nn_count != 1:
raise ex.InvalidComponentCountException("namenode", 1, nn_count)
dn_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "datanode")])
if dn_count < 1:
raise ex.InvalidComponentCountException("datanode", _("1 or more"),
nn_count)
rep_factor = utils.get_config_value_or_default('HDFS',
"dfs.replication",
cluster)
if dn_count < rep_factor:
raise ex.InvalidComponentCountException(
'datanode',
_('%s or more') % rep_factor, dn_count,
_('Number of %(dn)s instances should not be less '
'than %(replication)s') % {
'dn': 'datanode',
'replication': 'dfs.replication'
})
# validate Spark Master Node and Spark Slaves
sm_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "master")])
if sm_count < 1:
raise ex.RequiredServiceMissingException("Spark master")
if sm_count >= 2:
raise ex.InvalidComponentCountException("Spark master", "1",
sm_count)
sl_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "slave")])
if sl_count < 1:
raise ex.InvalidComponentCountException("Spark slave",
_("1 or more"), sl_count)
def validate(self, cluster):
nn_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "namenode")])
if nn_count != 1:
raise ex.InvalidComponentCountException("namenode", 1, nn_count)
jt_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "jobtracker")])
if jt_count > 1:
raise ex.InvalidComponentCountException("jobtracker", _('0 or 1'),
jt_count)
oozie_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "oozie")])
if oozie_count > 1:
raise ex.InvalidComponentCountException("oozie", _('0 or 1'),
oozie_count)
hive_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "hiveserver")])
if jt_count == 0:
tt_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "tasktracker")])
if tt_count > 0:
raise ex.RequiredServiceMissingException(
"jobtracker", required_by="tasktracker")
if oozie_count > 0:
raise ex.RequiredServiceMissingException(
"jobtracker", required_by="oozie")
if hive_count > 0:
raise ex.RequiredServiceMissingException(
"jobtracker", required_by="hive")
if hive_count > 1:
raise ex.InvalidComponentCountException("hive", _('0 or 1'),
hive_count)
def get_node_groups(self, node_process=None):
name = _get_node_process_name(node_process)
return u.get_node_groups(self.cluster, name)
def get_node_groups(self, node_process=None):
name = _get_node_process_name(node_process)
return u.get_node_groups(self.cluster, name)
def test_get_node_groups(self):
self.assertEqual(self.c1.node_groups, u.get_node_groups(self.c1))
self.assertEqual([], u.get_node_groups(self.c1, "wrong-process"))
self.assertEqual([self.ng2, self.ng3],
u.get_node_groups(self.c1, 'dn'))
def validate_scaling(self, cluster, existing, additional):
vl.validate_additional_ng_scaling(cluster, additional)
vl.validate_existing_ng_scaling(self.pctx, cluster, existing)
zk_ng = utils.get_node_groups(cluster, "zookeeper")
if zk_ng:
vl.validate_zookeeper_node_count(zk_ng, existing, additional)
def test_get_node_groups(self):
self.assertEqual(u.get_node_groups(self.c1), self.c1.node_groups)
self.assertEqual(u.get_node_groups(self.c1, "wrong-process"), [])
self.assertEqual(u.get_node_groups(self.c1, 'dn'),
[self.ng2, self.ng3])
def get_node_groups(self, node_process=None):
if node_process is not None:
node_process = su.get_node_process_name(node_process)
return u.get_node_groups(self.cluster, node_process)
def validate_cluster_creating(cluster):
mng_count = _get_inst_count(cluster, "CLOUDERA_MANAGER")
if mng_count != 1:
raise ex.InvalidComponentCountException("CLOUDERA_MANAGER", 1, mng_count)
zk_count = _get_inst_count(cluster, "ZOOKEEPER_SERVER")
nn_count = _get_inst_count(cluster, "HDFS_NAMENODE")
if nn_count != 1:
raise ex.InvalidComponentCountException("HDFS_NAMENODE", 1, nn_count)
snn_count = _get_inst_count(cluster, "HDFS_SECONDARYNAMENODE")
if snn_count != 1:
raise ex.InvalidComponentCountException("HDFS_SECONDARYNAMENODE", 1, snn_count)
dn_count = _get_inst_count(cluster, "HDFS_DATANODE")
replicas = PU.get_config_value("HDFS", "dfs_replication", cluster)
if dn_count < replicas:
raise ex.InvalidComponentCountException(
"HDFS_DATANODE", replicas, dn_count, _("Number of datanodes must be not less than dfs_replication.")
)
jn_count = _get_inst_count(cluster, "HDFS_JOURNALNODE")
require_anti_affinity = PU.c_helper.get_required_anti_affinity(cluster)
if jn_count > 0:
if jn_count < 3:
raise ex.InvalidComponentCountException("HDFS_JOURNALNODE", _("not less than 3"), jn_count)
if not jn_count % 2:
raise ex.InvalidComponentCountException("HDFS_JOURNALNODE", _("be odd"), jn_count)
if zk_count < 1:
raise ex.RequiredServiceMissingException("ZOOKEEPER", required_by="HDFS HA")
if require_anti_affinity:
if "HDFS_SECONDARYNAMENODE" not in _get_anti_affinity(cluster):
raise ex.NameNodeHAConfigurationError(
_("HDFS_SECONDARYNAMENODE should be enabled " "in anti_affinity.")
)
if "HDFS_NAMENODE" not in _get_anti_affinity(cluster):
raise ex.NameNodeHAConfigurationError(_("HDFS_NAMENODE should be enabled in anti_affinity."))
rm_count = _get_inst_count(cluster, "YARN_RESOURCEMANAGER")
if rm_count > 1:
raise ex.InvalidComponentCountException("YARN_RESOURCEMANAGER", _("0 or 1"), rm_count)
stdb_rm_count = _get_inst_count(cluster, "YARN_STANDBYRM")
if stdb_rm_count > 1:
raise ex.InvalidComponentCountException("YARN_STANDBYRM", _("0 or 1"), stdb_rm_count)
if stdb_rm_count > 0:
if rm_count < 1:
raise ex.RequiredServiceMissingException("YARN_RESOURCEMANAGER", required_by="RM HA")
if zk_count < 1:
raise ex.RequiredServiceMissingException("ZOOKEEPER", required_by="RM HA")
if require_anti_affinity:
if "YARN_RESOURCEMANAGER" not in _get_anti_affinity(cluster):
raise ex.ResourceManagerHAConfigurationError(
_("YARN_RESOURCEMANAGER should be enabled in " "anti_affinity.")
)
if "YARN_STANDBYRM" not in _get_anti_affinity(cluster):
raise ex.ResourceManagerHAConfigurationError(_("YARN_STANDBYRM should be enabled in anti_affinity."))
hs_count = _get_inst_count(cluster, "YARN_JOBHISTORY")
if hs_count > 1:
raise ex.InvalidComponentCountException("YARN_JOBHISTORY", _("0 or 1"), hs_count)
if rm_count > 0 and hs_count < 1:
raise ex.RequiredServiceMissingException("YARN_JOBHISTORY", required_by="YARN_RESOURCEMANAGER")
nm_count = _get_inst_count(cluster, "YARN_NODEMANAGER")
if rm_count == 0:
if nm_count > 0:
raise ex.RequiredServiceMissingException("YARN_RESOURCEMANAGER", required_by="YARN_NODEMANAGER")
oo_count = _get_inst_count(cluster, "OOZIE_SERVER")
if oo_count > 1:
raise ex.InvalidComponentCountException("OOZIE_SERVER", _("0 or 1"), oo_count)
if oo_count == 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException("HDFS_DATANODE", required_by="OOZIE_SERVER")
if nm_count < 1:
raise ex.RequiredServiceMissingException("YARN_NODEMANAGER", required_by="OOZIE_SERVER")
if hs_count != 1:
raise ex.RequiredServiceMissingException("YARN_JOBHISTORY", required_by="OOZIE_SERVER")
hms_count = _get_inst_count(cluster, "HIVE_METASTORE")
hvs_count = _get_inst_count(cluster, "HIVE_SERVER2")
whc_count = _get_inst_count(cluster, "HIVE_WEBHCAT")
if hms_count and rm_count < 1:
raise ex.RequiredServiceMissingException("YARN_RESOURCEMANAGER", required_by="HIVE_METASTORE")
if hms_count and not hvs_count:
raise ex.RequiredServiceMissingException("HIVE_SERVER2", required_by="HIVE_METASTORE")
if hvs_count and not hms_count:
raise ex.RequiredServiceMissingException("HIVE_METASTORE", required_by="HIVE_SERVER2")
if whc_count and not hms_count:
raise ex.RequiredServiceMissingException("HIVE_METASTORE", required_by="HIVE_WEBHCAT")
hue_count = _get_inst_count(cluster, "HUE_SERVER")
if hue_count > 1:
raise ex.InvalidComponentCountException("HUE_SERVER", _("0 or 1"), hue_count)
shs_count = _get_inst_count(cluster, "SPARK_YARN_HISTORY_SERVER")
if shs_count > 1:
raise ex.InvalidComponentCountException("SPARK_YARN_HISTORY_SERVER", _("0 or 1"), shs_count)
if shs_count and not rm_count:
raise ex.RequiredServiceMissingException("YARN_RESOURCEMANAGER", required_by="SPARK_YARN_HISTORY_SERVER")
if oo_count < 1 and hue_count:
raise ex.RequiredServiceMissingException("OOZIE_SERVER", required_by="HUE_SERVER")
if hms_count < 1 and hue_count:
raise ex.RequiredServiceMissingException("HIVE_METASTORE", required_by="HUE_SERVER")
hbm_count = _get_inst_count(cluster, "HBASE_MASTER")
hbr_count = _get_inst_count(cluster, "HBASE_REGIONSERVER")
if hbm_count >= 1:
if zk_count < 1:
raise ex.RequiredServiceMissingException("ZOOKEEPER", required_by="HBASE")
if hbr_count < 1:
raise ex.InvalidComponentCountException("HBASE_REGIONSERVER", _("at least 1"), hbr_count)
elif hbr_count >= 1:
raise ex.InvalidComponentCountException("HBASE_MASTER", _("at least 1"), hbm_count)
a_count = _get_inst_count(cluster, "FLUME_AGENT")
if a_count >= 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException("HDFS_DATANODE", required_by="FLUME_AGENT")
snt_count = _get_inst_count(cluster, "SENTRY_SERVER")
if snt_count > 1:
raise ex.InvalidComponentCountException("SENTRY_SERVER", _("0 or 1"), snt_count)
if snt_count == 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException("HDFS_DATANODE", required_by="SENTRY_SERVER")
if zk_count < 1:
raise ex.RequiredServiceMissingException("ZOOKEEPER", required_by="SENTRY_SERVER")
slr_count = _get_inst_count(cluster, "SOLR_SERVER")
if slr_count >= 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException("HDFS_DATANODE", required_by="SOLR_SERVER")
if zk_count < 1:
raise ex.RequiredServiceMissingException("ZOOKEEPER", required_by="SOLR_SERVER")
s2s_count = _get_inst_count(cluster, "SQOOP_SERVER")
if s2s_count > 1:
raise ex.InvalidComponentCountException("SQOOP_SERVER", _("0 or 1"), s2s_count)
if s2s_count == 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException("HDFS_DATANODE", required_by="SQOOP_SERVER")
if nm_count < 1:
raise ex.RequiredServiceMissingException("YARN_NODEMANAGER", required_by="SQOOP_SERVER")
if hs_count != 1:
raise ex.RequiredServiceMissingException("YARN_JOBHISTORY", required_by="SQOOP_SERVER")
lhbi_count = _get_inst_count(cluster, "HBASE_INDEXER")
if lhbi_count >= 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException("HDFS_DATANODE", required_by="HBASE_INDEXER")
if zk_count < 1:
raise ex.RequiredServiceMissingException("ZOOKEEPER", required_by="HBASE_INDEXER")
if slr_count < 1:
raise ex.RequiredServiceMissingException("SOLR_SERVER", required_by="HBASE_INDEXER")
if hbm_count < 1:
raise ex.RequiredServiceMissingException("HBASE_MASTER", required_by="HBASE_INDEXER")
ics_count = _get_inst_count(cluster, "IMPALA_CATALOGSERVER")
iss_count = _get_inst_count(cluster, "IMPALA_STATESTORE")
id_count = _get_inst_count(cluster, "IMPALAD")
if ics_count > 1:
raise ex.InvalidComponentCountException("IMPALA_CATALOGSERVER", _("0 or 1"), ics_count)
if iss_count > 1:
raise ex.InvalidComponentCountException("IMPALA_STATESTORE", _("0 or 1"), iss_count)
if ics_count == 1:
datanode_ng = u.get_node_groups(cluster, "HDFS_DATANODE")
impalad_ng = u.get_node_groups(cluster, "IMPALAD")
datanodes = set(ng.id for ng in datanode_ng)
impalads = set(ng.id for ng in impalad_ng)
if datanodes != impalads:
raise ex.InvalidClusterTopology(_("IMPALAD must be installed on every HDFS_DATANODE"))
if iss_count != 1:
raise ex.RequiredServiceMissingException("IMPALA_STATESTORE", required_by="IMPALA")
if id_count < 1:
raise ex.RequiredServiceMissingException("IMPALAD", required_by="IMPALA")
if dn_count < 1:
raise ex.RequiredServiceMissingException("HDFS_DATANODE", required_by="IMPALA")
if hms_count < 1:
raise ex.RequiredServiceMissingException("HIVE_METASTORE", required_by="IMPALA")
kms_count = _get_inst_count(cluster, "KMS")
if kms_count > 1:
raise ex.InvalidComponentCountException("KMS", _("0 or 1"), kms_count)
def get_node_groups(self, node_process=None):
return u.get_node_groups(self.cluster, node_process)
def validate_cluster_creating(cluster):
mng_count = _get_inst_count(cluster, 'CLOUDERA_MANAGER')
if mng_count != 1:
raise ex.InvalidComponentCountException('CLOUDERA_MANAGER',
1, mng_count)
nn_count = _get_inst_count(cluster, 'HDFS_NAMENODE')
if nn_count != 1:
raise ex.InvalidComponentCountException('HDFS_NAMENODE', 1, nn_count)
snn_count = _get_inst_count(cluster, 'HDFS_SECONDARYNAMENODE')
if snn_count != 1:
raise ex.InvalidComponentCountException('HDFS_SECONDARYNAMENODE', 1,
snn_count)
dn_count = _get_inst_count(cluster, 'HDFS_DATANODE')
replicas = PU.get_config_value('HDFS', 'dfs_replication', cluster)
if dn_count < replicas:
raise ex.InvalidComponentCountException(
'HDFS_DATANODE', replicas, dn_count,
_('Number of datanodes must be not less than dfs_replication.'))
rm_count = _get_inst_count(cluster, 'YARN_RESOURCEMANAGER')
if rm_count > 1:
raise ex.InvalidComponentCountException('YARN_RESOURCEMANAGER',
_('0 or 1'), rm_count)
hs_count = _get_inst_count(cluster, 'YARN_JOBHISTORY')
if hs_count > 1:
raise ex.InvalidComponentCountException('YARN_JOBHISTORY',
_('0 or 1'), hs_count)
if rm_count > 0 and hs_count < 1:
raise ex.RequiredServiceMissingException(
'YARN_JOBHISTORY', required_by='YARN_RESOURCEMANAGER')
nm_count = _get_inst_count(cluster, 'YARN_NODEMANAGER')
if rm_count == 0:
if nm_count > 0:
raise ex.RequiredServiceMissingException(
'YARN_RESOURCEMANAGER', required_by='YARN_NODEMANAGER')
oo_count = _get_inst_count(cluster, 'OOZIE_SERVER')
if oo_count > 1:
raise ex.InvalidComponentCountException('OOZIE_SERVER', _('0 or 1'),
oo_count)
if oo_count == 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException(
'HDFS_DATANODE', required_by='OOZIE_SERVER')
if nm_count < 1:
raise ex.RequiredServiceMissingException(
'YARN_NODEMANAGER', required_by='OOZIE_SERVER')
if hs_count != 1:
raise ex.RequiredServiceMissingException(
'YARN_JOBHISTORY', required_by='OOZIE_SERVER')
hms_count = _get_inst_count(cluster, 'HIVE_METASTORE')
hvs_count = _get_inst_count(cluster, 'HIVE_SERVER2')
whc_count = _get_inst_count(cluster, 'HIVE_WEBHCAT')
if hms_count and rm_count < 1:
raise ex.RequiredServiceMissingException(
'YARN_RESOURCEMANAGER', required_by='HIVE_METASTORE')
if hms_count and not hvs_count:
raise ex.RequiredServiceMissingException(
'HIVE_SERVER2', required_by='HIVE_METASTORE')
if hvs_count and not hms_count:
raise ex.RequiredServiceMissingException(
'HIVE_METASTORE', required_by='HIVE_SERVER2')
if whc_count and not hms_count:
raise ex.RequiredServiceMissingException(
'HIVE_METASTORE', required_by='HIVE_WEBHCAT')
hue_count = _get_inst_count(cluster, 'HUE_SERVER')
if hue_count > 1:
raise ex.InvalidComponentCountException('HUE_SERVER', _('0 or 1'),
hue_count)
shs_count = _get_inst_count(cluster, 'SPARK_YARN_HISTORY_SERVER')
if shs_count > 1:
raise ex.InvalidComponentCountException('SPARK_YARN_HISTORY_SERVER',
_('0 or 1'), shs_count)
if shs_count and not rm_count:
raise ex.RequiredServiceMissingException(
'YARN_RESOURCEMANAGER', required_by='SPARK_YARN_HISTORY_SERVER')
if oo_count < 1 and hue_count:
raise ex.RequiredServiceMissingException(
'OOZIE_SERVER', required_by='HUE_SERVER')
if hms_count < 1 and hue_count:
raise ex.RequiredServiceMissingException(
'HIVE_METASTORE', required_by='HUE_SERVER')
hbm_count = _get_inst_count(cluster, 'HBASE_MASTER')
hbr_count = _get_inst_count(cluster, 'HBASE_REGIONSERVER')
zk_count = _get_inst_count(cluster, 'ZOOKEEPER_SERVER')
if hbm_count >= 1:
if zk_count < 1:
raise ex.RequiredServiceMissingException('ZOOKEEPER',
required_by='HBASE')
if hbr_count < 1:
raise ex.InvalidComponentCountException(
'HBASE_REGIONSERVER', _('at least 1'), hbr_count)
elif hbr_count >= 1:
raise ex.InvalidComponentCountException('HBASE_MASTER',
_('at least 1'), hbm_count)
a_count = _get_inst_count(cluster, 'FLUME_AGENT')
if a_count >= 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException(
'HDFS_DATANODE', required_by='FLUME_AGENT')
snt_count = _get_inst_count(cluster, 'SENTRY_SERVER')
if snt_count > 1:
raise ex.InvalidComponentCountException('SENTRY_SERVER', _('0 or 1'),
snt_count)
if snt_count == 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException(
'HDFS_DATANODE', required_by='SENTRY_SERVER')
if zk_count < 1:
raise ex.RequiredServiceMissingException(
'ZOOKEEPER', required_by='SENTRY_SERVER')
slr_count = _get_inst_count(cluster, 'SOLR_SERVER')
if slr_count >= 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException(
'HDFS_DATANODE', required_by='SOLR_SERVER')
if zk_count < 1:
raise ex.RequiredServiceMissingException(
'ZOOKEEPER', required_by='SOLR_SERVER')
s2s_count = _get_inst_count(cluster, 'SQOOP_SERVER')
if s2s_count > 1:
raise ex.InvalidComponentCountException('SQOOP_SERVER', _('0 or 1'),
s2s_count)
if s2s_count == 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException(
'HDFS_DATANODE', required_by='SQOOP_SERVER')
if nm_count < 1:
raise ex.RequiredServiceMissingException(
'YARN_NODEMANAGER', required_by='SQOOP_SERVER')
if hs_count != 1:
raise ex.RequiredServiceMissingException(
'YARN_JOBHISTORY', required_by='SQOOP_SERVER')
lhbi_count = _get_inst_count(cluster, 'HBASE_INDEXER')
if lhbi_count >= 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException(
'HDFS_DATANODE', required_by='HBASE_INDEXER')
if zk_count < 1:
raise ex.RequiredServiceMissingException(
'ZOOKEEPER', required_by='HBASE_INDEXER')
if slr_count < 1:
raise ex.RequiredServiceMissingException(
'SOLR_SERVER', required_by='HBASE_INDEXER')
if hbm_count < 1:
raise ex.RequiredServiceMissingException(
'HBASE_MASTER', required_by='HBASE_INDEXER')
ics_count = _get_inst_count(cluster, 'IMPALA_CATALOGSERVER')
iss_count = _get_inst_count(cluster, 'IMPALA_STATESTORE')
id_count = _get_inst_count(cluster, 'IMPALAD')
if ics_count > 1:
raise ex.InvalidComponentCountException('IMPALA_CATALOGSERVER',
_('0 or 1'), ics_count)
if iss_count > 1:
raise ex.InvalidComponentCountException('IMPALA_STATESTORE',
_('0 or 1'), iss_count)
if ics_count == 1:
datanode_ng = u.get_node_groups(cluster, "HDFS_DATANODE")
impalad_ng = u.get_node_groups(cluster, "IMPALAD")
datanodes = set(ng.id for ng in datanode_ng)
impalads = set(ng.id for ng in impalad_ng)
if datanodes != impalads:
raise ex.InvalidClusterTopology(
_("IMPALAD must be installed on every HDFS_DATANODE"))
if iss_count != 1:
raise ex.RequiredServiceMissingException(
'IMPALA_STATESTORE', required_by='IMPALA')
if id_count < 1:
raise ex.RequiredServiceMissingException(
'IMPALAD', required_by='IMPALA')
if dn_count < 1:
raise ex.RequiredServiceMissingException(
'HDFS_DATANODE', required_by='IMPALA')
if hms_count < 1:
raise ex.RequiredServiceMissingException(
'HIVE_METASTORE', required_by='IMPALA')
def test_get_node_groups(self):
self.assertEqual(u.get_node_groups(self.c1), self.c1.node_groups)
self.assertEqual(u.get_node_groups(self.c1, "wrong-process"), [])
self.assertEqual(u.get_node_groups(self.c1, 'dn'),
[self.ng2, self.ng3])
def get_node_groups(self, node_process=None):
if node_process is not None:
node_process = su.get_node_process_name(node_process)
return u.get_node_groups(self.cluster, node_process)
def _get_inst_count(cluster, process):
return sum([ng.count for ng in u.get_node_groups(cluster, process)])
def test_get_node_groups(self):
self.assertEqual(self.c1.node_groups, u.get_node_groups(self.c1))
self.assertEqual([], u.get_node_groups(self.c1, "wrong-process"))
self.assertEqual([self.ng2, self.ng3],
u.get_node_groups(self.c1, 'dn'))
def get_inst_count(cls, cluster, process):
return sum([ng.count for ng in u.get_node_groups(cluster, process)])
def get_node_groups(self, node_process=None):
return u.get_node_groups(self.cluster, node_process)
