def validate(self, cluster):
nn_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "namenode")])
if nn_count != 1:
raise ex.InvalidComponentCountException("namenode", 1, nn_count)
dn_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "datanode")])
if dn_count < 1:
raise ex.InvalidComponentCountException("datanode", _("1 or more"),
nn_count)
# validate Spark Master Node and Spark Slaves
sm_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "master")])
if sm_count != 1:
raise ex.RequiredServiceMissingException("Spark master")
sl_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "slave")])
if sl_count < 1:
raise ex.InvalidComponentCountException("Spark slave",
_("1 or more"), sl_count)
def validate(self, cluster_spec, cluster):
count = cluster_spec.get_deployed_node_group_count('OOZIE_SERVER')
if count != 1:
raise ex.InvalidComponentCountException('OOZIE_SERVER', 1, count)
count = cluster_spec.get_deployed_node_group_count('OOZIE_CLIENT')
if not count:
raise ex.InvalidComponentCountException('OOZIE_CLIENT', '1+',
count)
def validate(self, cluster_spec, cluster):
count = cluster_spec.get_deployed_node_group_count('JOBTRACKER')
if count != 1:
raise ex.InvalidComponentCountException('JOBTRACKER', 1, count)
count = cluster_spec.get_deployed_node_group_count('TASKTRACKER')
if not count:
raise ex.InvalidComponentCountException('TASKTRACKER', '> 0',
count)
def validate(self, cluster_spec, cluster):
count = cluster_spec.get_deployed_node_group_count('RESOURCEMANAGER')
if count != 1:
raise ex.InvalidComponentCountException('RESOURCEMANAGER', 1,
count)
count = cluster_spec.get_deployed_node_group_count('NODEMANAGER')
if not count:
raise ex.InvalidComponentCountException('NODEMANAGER', '> 0',
count)
def validate_job_execution(self, cluster, job, data):
oo_count = u.get_instances_count(cluster, 'OOZIE_SERVER')
if oo_count != 1:
raise ex.InvalidComponentCountException(
'OOZIE_SERVER', '1', oo_count)
super(EdpOozieEngine, self).validate_job_execution(cluster, job, data)
def validate_cluster_creating(cluster):
nn_count = _get_inst_count(cluster, 'namenode')
if nn_count != 1:
raise ex.InvalidComponentCountException('namenode', 1, nn_count)
snn_count = _get_inst_count(cluster, 'secondarynamenode')
if snn_count not in [0, 1]:
raise ex.InvalidComponentCountException('secondarynamenode', '0 or 1',
snn_count)
rm_count = _get_inst_count(cluster, 'resourcemanager')
if rm_count not in [0, 1]:
raise ex.InvalidComponentCountException('resourcemanager', '0 or 1',
rm_count)
hs_count = _get_inst_count(cluster, 'historyserver')
if hs_count not in [0, 1]:
raise ex.InvalidComponentCountException('historyserver', '0 or 1',
hs_count)
nm_count = _get_inst_count(cluster, 'nodemanager')
if rm_count == 0:
if nm_count > 0:
raise ex.RequiredServiceMissingException('resourcemanager',
required_by='nodemanager')
oo_count = _get_inst_count(cluster, 'oozie')
dn_count = _get_inst_count(cluster, 'datanode')
if oo_count not in [0, 1]:
raise ex.InvalidComponentCountException('oozie', '0 or 1', oo_count)
if oo_count == 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException('datanode',
required_by='oozie')
if nm_count < 1:
raise ex.RequiredServiceMissingException('nodemanager',
required_by='oozie')
if hs_count != 1:
raise ex.RequiredServiceMissingException('historyserver',
required_by='oozie')
def validate(self, cluster):
nn_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "namenode")])
if nn_count != 1:
raise ex.InvalidComponentCountException("namenode", 1, nn_count)
jt_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "jobtracker")])
if jt_count not in [0, 1]:
raise ex.InvalidComponentCountException("jobtracker", _('0 or 1'),
jt_count)
oozie_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "oozie")])
if oozie_count not in [0, 1]:
raise ex.InvalidComponentCountException("oozie", _('0 or 1'),
oozie_count)
hive_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "hiveserver")])
if jt_count == 0:
tt_count = sum([
ng.count
for ng in utils.get_node_groups(cluster, "tasktracker")
])
if tt_count > 0:
raise ex.RequiredServiceMissingException(
"jobtracker", required_by="tasktracker")
if oozie_count > 0:
raise ex.RequiredServiceMissingException("jobtracker",
required_by="oozie")
if hive_count > 0:
raise ex.RequiredServiceMissingException("jobtracker",
required_by="hive")
if hive_count not in [0, 1]:
raise ex.InvalidComponentCountException("hive", _('0 or 1'),
hive_count)
def validate(self, cluster):
nn_count = sum(
[ng.count for ng in u.get_node_groups(cluster, 'namenode')])
if nn_count != 1:
raise ex.InvalidComponentCountException('namenode', 1, nn_count)
jt_count = sum(
[ng.count for ng in u.get_node_groups(cluster, 'jobtracker')])
if jt_count > 1:
raise ex.InvalidComponentCountException('jobtracker', '0 or 1',
jt_count)
tt_count = sum(
[ng.count for ng in u.get_node_groups(cluster, 'tasktracker')])
if jt_count == 0 and tt_count > 0:
raise ex.RequiredServiceMissingException('jobtracker',
required_by='tasktracker')
mng_count = sum(
[ng.count for ng in u.get_node_groups(cluster, 'manager')])
if mng_count != 1:
raise ex.InvalidComponentCountException('manager', 1, mng_count)
def validate_cluster_creating(cluster):
if not cmu.have_cm_api_libs():
LOG.error(
_LE("For provisioning cluster with CDH plugin install"
"'cm_api' package version 6.0.2 or later."))
raise ex.HadoopProvisionError(_("'cm_api' is not installed."))
mng_count = _get_inst_count(cluster, 'MANAGER')
if mng_count != 1:
raise ex.InvalidComponentCountException('MANAGER', 1, mng_count)
nn_count = _get_inst_count(cluster, 'NAMENODE')
if nn_count != 1:
raise ex.InvalidComponentCountException('NAMENODE', 1, nn_count)
snn_count = _get_inst_count(cluster, 'SECONDARYNAMENODE')
if snn_count != 1:
raise ex.InvalidComponentCountException('SECONDARYNAMENODE', 1,
snn_count)
rm_count = _get_inst_count(cluster, 'RESOURCEMANAGER')
if rm_count not in [0, 1]:
raise ex.InvalidComponentCountException('RESOURCEMANAGER', '0 or 1',
rm_count)
hs_count = _get_inst_count(cluster, 'JOBHISTORY')
if hs_count not in [0, 1]:
raise ex.InvalidComponentCountException('JOBHISTORY', '0 or 1',
hs_count)
if rm_count > 0 and hs_count < 1:
raise ex.RequiredServiceMissingException('JOBHISTORY',
required_by='RESOURCEMANAGER')
nm_count = _get_inst_count(cluster, 'NODEMANAGER')
if rm_count == 0:
if nm_count > 0:
raise ex.RequiredServiceMissingException('RESOURCEMANAGER',
required_by='NODEMANAGER')
oo_count = _get_inst_count(cluster, 'OOZIE_SERVER')
dn_count = _get_inst_count(cluster, 'DATANODE')
if oo_count not in [0, 1]:
raise ex.InvalidComponentCountException('OOZIE_SERVER', '0 or 1',
oo_count)
if oo_count == 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException(
'DATANODE', required_by='OOZIE_SERVER')
if nm_count < 1:
raise ex.RequiredServiceMissingException(
'NODEMANAGER', required_by='OOZIE_SERVER')
if hs_count != 1:
raise ex.RequiredServiceMissingException(
'JOBHISTORY', required_by='OOZIE_SERVER')
def validate_cluster_creating(pctx, cluster):
nn_count = _get_inst_count(cluster, 'namenode')
if nn_count != 1:
raise ex.InvalidComponentCountException('namenode', 1, nn_count)
snn_count = _get_inst_count(cluster, 'secondarynamenode')
if snn_count not in [0, 1]:
raise ex.InvalidComponentCountException('secondarynamenode', '0 or 1',
snn_count)
rm_count = _get_inst_count(cluster, 'resourcemanager')
if rm_count not in [0, 1]:
raise ex.InvalidComponentCountException('resourcemanager', '0 or 1',
rm_count)
hs_count = _get_inst_count(cluster, 'historyserver')
if hs_count not in [0, 1]:
raise ex.InvalidComponentCountException('historyserver', '0 or 1',
hs_count)
nm_count = _get_inst_count(cluster, 'nodemanager')
if rm_count == 0:
if nm_count > 0:
raise ex.RequiredServiceMissingException('resourcemanager',
required_by='nodemanager')
oo_count = _get_inst_count(cluster, 'oozie')
dn_count = _get_inst_count(cluster, 'datanode')
if oo_count not in [0, 1]:
raise ex.InvalidComponentCountException('oozie', '0 or 1', oo_count)
if oo_count == 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException('datanode',
required_by='oozie')
if nm_count < 1:
raise ex.RequiredServiceMissingException('nodemanager',
required_by='oozie')
if hs_count != 1:
raise ex.RequiredServiceMissingException('historyserver',
required_by='oozie')
rep_factor = cu.get_config_value(pctx, 'HDFS', 'dfs.replication', cluster)
if dn_count < rep_factor:
raise ex.InvalidComponentCountException(
'datanode', rep_factor, dn_count,
_('Number of datanodes must be '
'not less than '
'dfs.replication.'))
def validate(self, cluster_spec, cluster):
count = cluster_spec.get_deployed_node_group_count('HUE')
if count != 1:
raise ex.InvalidComponentCountException('HUE', 1, count)
services = cluster_spec.services
for reqd_service in self.required_services:
reqd_service_deployed = False
if services is not None:
for service in services:
reqd_service_deployed = (service.deployed
and service.name == reqd_service)
if reqd_service_deployed:
break
if not reqd_service_deployed:
raise ex.RequiredServiceMissingException(
reqd_service, self.name)
def validate(self, cluster_spec, cluster):
count = cluster_spec.get_deployed_node_group_count('ZOOKEEPER_SERVER')
if count < 1:
raise ex.InvalidComponentCountException('ZOOKEEPER_SERVER', '1+',
count)
def validate(self, cluster_spec, cluster):
# check for a single HBASE_SERVER
count = cluster_spec.get_deployed_node_group_count('HBASE_MASTER')
if count != 1:
raise ex.InvalidComponentCountException('HBASE_MASTER', 1, count)
def validate(self, cluster_spec, cluster):
# check for a single NAMENODE
count = cluster_spec.get_deployed_node_group_count('NAMENODE')
if count != 1:
raise ex.InvalidComponentCountException('NAMENODE', 1, count)
def get_instance(cluster, node_process):
instances = get_instances(cluster, node_process)
if len(instances) > 1:
raise ex.InvalidComponentCountException(node_process, '0 or 1',
len(instances))
return instances[0] if instances else None
def validate_edp(self, cluster):
oo_count = u.get_instances_count(cluster, 'oozie_server')
if oo_count != 1:
raise ex.InvalidComponentCountException('oozie', '1', oo_count)
def validate(self, cluster_spec, cluster):
count = cluster_spec.get_deployed_node_group_count('AMBARI_SERVER')
if count != 1:
raise ex.InvalidComponentCountException('AMBARI_SERVER', 1, count)
def validate_edp(self, cluster):
oo_count = u.get_instances_count(cluster, 'OOZIE_SERVER')
if oo_count != 1:
raise ex.InvalidComponentCountException('OOZIE_SERVER', '1',
oo_count)
