def test_instances_with_services(self):
inst = [FakeInstace(["1", "2", "3"]), FakeInstace(["1", "3"]),
FakeInstace(["1"]), FakeInstace(["3"])]
self.assertEqual(4, len(pu.instances_with_services(inst, ["1", "3"])))
self.assertEqual(1, len(pu.instances_with_services(inst, ["2"])))
self.assertEqual(3, len(pu.instances_with_services(inst, ["3"])))
self.assertEqual(0, len(pu.instances_with_services(inst, ["5"])))
def test_instances_with_services(self):
inst = [FakeInstance("1", ["nodeprocess1"]),
FakeInstance("2", ["nodeprocess2"])]
node_processes = ["nodeprocess"]
res = pu.instances_with_services(inst, node_processes)
self.assertEqual([], res)
node_processes = ["nodeprocess1"]
res = pu.instances_with_services(inst, node_processes)
self.assertEqual([FakeInstance("1", ["nodeprocess1"])], res)
def test_instances_with_services(self):
inst = [
FakeInstace(["1", "2", "3"]),
FakeInstace(["1", "3"]),
FakeInstace(["1"]),
FakeInstace(["3"])
]
self.assertEqual(4, len(pu.instances_with_services(inst, ["1", "3"])))
self.assertEqual(1, len(pu.instances_with_services(inst, ["2"])))
self.assertEqual(3, len(pu.instances_with_services(inst, ["3"])))
self.assertEqual(0, len(pu.instances_with_services(inst, ["5"])))
def deploy_kerberos_principals(cluster, instances=None):
if not kerberos.is_kerberos_security_enabled(cluster):
return
if instances is None:
instances = plugin_utils.get_instances(cluster)
mapper = {
'hdfs':
plugin_utils.instances_with_services(instances, [
p_common.SECONDARY_NAMENODE, p_common.NAMENODE, p_common.DATANODE,
p_common.JOURNAL_NODE
]),
'spark':
plugin_utils.instances_with_services(
instances, [p_common.SPARK_JOBHISTORYSERVER]),
'oozie':
plugin_utils.instances_with_services(instances,
[p_common.OOZIE_SERVER]),
}
kerberos.create_keytabs_for_map(cluster, mapper)
def _start_tt_dn_processes(self, instances):
tt_dn_names = ["datanode", "tasktracker"]
instances = utils.instances_with_services(instances, tt_dn_names)
if not instances:
return
cpo.add_provisioning_step(
instances[0].cluster_id, utils.start_process_event_message("DataNodes, TaskTrackers"), len(instances)
)
with context.ThreadGroup() as tg:
for i in instances:
processes = set(i.node_group.node_processes)
tt_dn_procs = processes.intersection(tt_dn_names)
tg.spawn("vanilla-start-tt-dn-%s" % i.instance_name, self._start_tt_dn, i, list(tt_dn_procs))
def start_dn_nm_processes(instances):
filternames = ['datanode', 'nodemanager']
instances = pu.instances_with_services(instances, filternames)
if len(instances) == 0:
return
cpo.add_provisioning_step(
instances[0].cluster_id,
pu.start_process_event_message("DataNodes, NodeManagers"),
len(instances))
with context.ThreadGroup() as tg:
for instance in instances:
processes = set(instance.node_group.node_processes)
processes = processes.intersection(filternames)
tg.spawn('vanilla-start-processes-%s' % instance.instance_name,
_start_processes, instance, list(processes))
def _start_tt_dn_processes(self, instances):
tt_dn_names = ["datanode", "tasktracker"]
instances = utils.instances_with_services(instances, tt_dn_names)
if not instances:
return
cpo.add_provisioning_step(
instances[0].cluster_id,
utils.start_process_event_message("DataNodes, TaskTrackers"),
len(instances))
with context.ThreadGroup() as tg:
for i in instances:
processes = set(i.node_group.node_processes)
tt_dn_procs = processes.intersection(tt_dn_names)
tg.spawn('vanilla-start-tt-dn-%s' % i.instance_name,
self._start_tt_dn, i, list(tt_dn_procs))
def start_dn_nm_processes(instances):
filternames = ['datanode', 'nodemanager']
instances = pu.instances_with_services(instances, filternames)
if len(instances) == 0:
return
cpo.add_provisioning_step(
instances[0].cluster_id,
pu.start_process_event_message("DataNodes, NodeManagers"),
len(instances))
with context.ThreadGroup() as tg:
for instance in instances:
with context.set_current_instance_id(instance.instance_id):
processes = set(instance.node_group.node_processes)
processes = processes.intersection(filternames)
tg.spawn('vanilla-start-processes-%s' % instance.instance_name,
_start_processes, instance, list(processes))
def get_hdfs_nodes(self, cluster, instances=None):
instances = instances if instances else u.get_instances(cluster)
return u.instances_with_services(
instances, ["HDFS_DATANODE", "HDFS_NAMENODE",
"HDFS_SECONDARYNAMENODE"])
