def test_get_nodemanagers(self):
cl = tu.create_cluster('cl1', 't1', 'vanilla', '2.7.1',
[self.ng_manager, self.ng_nodemanager])
nodemanagers = u.get_nodemanagers(cl)
self.assertEqual(2, len(nodemanagers))
self.assertEqual(
set(['tt1', 'tt2']),
set([nodemanagers[0].instance_id, nodemanagers[1].instance_id]))
cl = tu.create_cluster('cl1', 't1', 'vanilla', '2.7.1',
[self.ng_namenode])
self.assertEqual([], u.get_nodemanagers(cl))
def start_cluster(self, cluster):
nn = vu.get_namenode(cluster)
run.format_namenode(nn)
run.start_hadoop_process(nn, "namenode")
for snn in vu.get_secondarynamenodes(cluster):
run.start_hadoop_process(snn, "secondarynamenode")
rm = vu.get_resourcemanager(cluster)
if rm:
run.start_yarn_process(rm, "resourcemanager")
for dn in vu.get_datanodes(cluster):
run.start_hadoop_process(dn, "datanode")
run.await_datanodes(cluster)
for nm in vu.get_nodemanagers(cluster):
run.start_yarn_process(nm, "nodemanager")
hs = vu.get_historyserver(cluster)
if hs:
run.start_historyserver(hs)
oo = vu.get_oozie(cluster)
if oo:
run.start_oozie_process(oo)
self._set_cluster_info(cluster)
def start_cluster(self, cluster):
nn = vu.get_namenode(cluster)
run.format_namenode(nn)
run.start_hadoop_process(nn, 'namenode')
for snn in vu.get_secondarynamenodes(cluster):
run.start_hadoop_process(snn, 'secondarynamenode')
rm = vu.get_resourcemanager(cluster)
run.start_yarn_process(rm, 'resourcemanager')
for dn in vu.get_datanodes(cluster):
run.start_hadoop_process(dn, 'datanode')
run.await_datanodes(cluster)
for nm in vu.get_nodemanagers(cluster):
run.start_yarn_process(nm, 'nodemanager')
hs = vu.get_historyserver(cluster)
if hs:
run.start_historyserver(hs)
oo = vu.get_oozie(cluster)
if oo:
run.start_oozie_process(oo)
self._set_cluster_info(cluster)
def _update_include_files(cluster):
instances = u.get_instances(cluster)
datanodes = vu.get_datanodes(cluster)
nodemanagers = vu.get_nodemanagers(cluster)
dn_hosts = u.generate_fqdn_host_names(datanodes)
nm_hosts = u.generate_fqdn_host_names(nodemanagers)
for instance in instances:
with instance.remote() as r:
r.execute_command(
'sudo su - -c "echo \'%s\' > %s/dn-include" hadoop' % (
dn_hosts, HADOOP_CONF_DIR))
r.execute_command(
'sudo su - -c "echo \'%s\' > %s/nm-include" hadoop' % (
nm_hosts, HADOOP_CONF_DIR))
def _update_include_files(cluster):
instances = u.get_instances(cluster)
datanodes = vu.get_datanodes(cluster)
nodemanagers = vu.get_nodemanagers(cluster)
dn_hosts = u.generate_fqdn_host_names(datanodes)
nm_hosts = u.generate_fqdn_host_names(nodemanagers)
for instance in instances:
with instance.remote() as r:
r.execute_command(
'sudo su - -c "echo \'%s\' > %s/dn-include" hadoop' %
(dn_hosts, HADOOP_CONF_DIR))
r.execute_command(
'sudo su - -c "echo \'%s\' > %s/nm-include" hadoop' %
(nm_hosts, HADOOP_CONF_DIR))
def _update_include_files(cluster, dec_instances=None):
dec_instances = dec_instances or []
dec_instances_ids = [instance.id for instance in dec_instances]
instances = u.get_instances(cluster)
inst_filter = lambda inst: inst.id not in dec_instances_ids
datanodes = filter(inst_filter, vu.get_datanodes(cluster))
nodemanagers = filter(inst_filter, vu.get_nodemanagers(cluster))
dn_hosts = u.generate_fqdn_host_names(datanodes)
nm_hosts = u.generate_fqdn_host_names(nodemanagers)
for instance in instances:
with instance.remote() as r:
r.execute_command("sudo su - -c \"echo '%s' > %s/dn-include\" hadoop" % (dn_hosts, HADOOP_CONF_DIR))
r.execute_command("sudo su - -c \"echo '%s' > %s/nm-include\" hadoop" % (nm_hosts, HADOOP_CONF_DIR))
def _update_include_files(cluster, dec_instances=None):
dec_instances = dec_instances or []
dec_instances_ids = [instance.id for instance in dec_instances]
instances = u.get_instances(cluster)
inst_filter = lambda inst: inst.id not in dec_instances_ids
datanodes = filter(inst_filter, vu.get_datanodes(cluster))
nodemanagers = filter(inst_filter, vu.get_nodemanagers(cluster))
dn_hosts = u.generate_fqdn_host_names(datanodes)
nm_hosts = u.generate_fqdn_host_names(nodemanagers)
for instance in instances:
with instance.remote() as r:
r.execute_command(
'sudo su - -c "echo \'%s\' > %s/dn-include" hadoop' %
(dn_hosts, HADOOP_CONF_DIR))
r.execute_command(
'sudo su - -c "echo \'%s\' > %s/nm-include" hadoop' %
(nm_hosts, HADOOP_CONF_DIR))
