def _set_cluster_info(self, cluster):
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
oozie = utils.get_oozie(cluster)
info = {}
if jt:
address = c_helper.get_config_value(
'MapReduce', 'mapred.job.tracker.http.address', cluster)
port = address[address.rfind(':') + 1:]
info['MapReduce'] = {
'Web UI': 'http://%s:%s' % (jt.management_ip, port)
}
if nn:
address = c_helper.get_config_value('HDFS', 'dfs.http.address',
cluster)
port = address[address.rfind(':') + 1:]
info['HDFS'] = {
'Web UI': 'http://%s:%s' % (nn.management_ip, port)
}
if oozie:
info['JobFlow'] = {
'Oozie': 'http://%s:11000' % oozie.management_ip
}
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {'info': info})
def _set_cluster_info(self, cluster):
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
oozie = utils.get_oozie(cluster)
info = {}
if jt:
address = c_helper.get_config_value(
'MapReduce', 'mapred.job.tracker.http.address', cluster)
port = address[address.rfind(':') + 1:]
info['MapReduce'] = {
'Web UI': 'http://%s:%s' % (jt.management_ip, port)
}
#TODO(aignatov) change from hardcode value
info['MapReduce']['JobTracker'] = '%s:8021' % jt.hostname()
if nn:
address = c_helper.get_config_value(
'HDFS', 'dfs.http.address', cluster)
port = address[address.rfind(':') + 1:]
info['HDFS'] = {
'Web UI': 'http://%s:%s' % (nn.management_ip, port)
}
#TODO(aignatov) change from hardcode value
info['HDFS']['NameNode'] = 'hdfs://%s:8020' % nn.hostname()
if oozie:
info['JobFlow'] = {
'Oozie': 'http://%s:11000' % oozie.management_ip
}
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {'info': info})
def _set_cluster_info(self, cluster):
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
oozie = utils.get_oozie(cluster)
info = {}
if jt:
address = c_helper.get_config_value(
'MapReduce', 'mapred.job.tracker.http.address', cluster)
port = address[address.rfind(':') + 1:]
info['MapReduce'] = {
'Web UI': 'http://%s:%s' % (jt.management_ip, port)
}
if nn:
address = c_helper.get_config_value(
'HDFS', 'dfs.http.address', cluster)
port = address[address.rfind(':') + 1:]
info['HDFS'] = {
'Web UI': 'http://%s:%s' % (nn.management_ip, port)
}
if oozie:
info['JobFlow'] = {
'Oozie': 'http://%s:11000' % oozie.management_ip
}
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {'info': info})
def test_get_config_value(self, cond_get_cluster):
cluster = self._get_fake_cluster()
cond_get_cluster.return_value = cluster
self.assertEqual(
c_h.get_config_value('HDFS', 'fs.default.name', cluster),
'hdfs://inst1:8020')
self.assertEqual(
c_h.get_config_value('HDFS', 'spam', cluster), 'eggs')
self.assertEqual(
c_h.get_config_value('HDFS', 'dfs.safemode.extension'), 30000)
self.assertRaises(RuntimeError,
c_h.get_config_value,
'MapReduce', 'spam', cluster)
def _validate_existing_ng_scaling(self, cluster, existing):
scalable_processes = self._get_scalable_processes()
dn_to_delete = 0
for ng in cluster.node_groups:
if ng.id in existing:
if ng.count > existing[ng.id] and "datanode" in \
ng.node_processes:
dn_to_delete += ng.count - existing[ng.id]
if not set(ng.node_processes).issubset(scalable_processes):
raise ex.NodeGroupCannotBeScaled(
ng.name, "Vanilla plugin cannot scale nodegroup"
" with processes: " +
' '.join(ng.node_processes))
dn_amount = len(utils.get_datanodes(cluster))
rep_factor = c_helper.get_config_value('HDFS', 'dfs.replication',
cluster)
if dn_to_delete > 0 and dn_amount - dn_to_delete < rep_factor:
raise ex.ClusterCannotBeScaled(
cluster.name, "Vanilla plugin cannot shrink cluster because "
"it would be not enough nodes for replicas "
"(replication factor is %s)" % rep_factor)
def test_get_config_value(self, cond_get_cluster):
class FakeNG(object):
def __init__(self, name, flavor, processes, count, instances=None,
configuration=None, cluster_id=None):
self.name = name
self.flavor = flavor
self.node_processes = processes
self.count = count
self.instances = instances or []
self.ng_configuration = configuration
self.cluster_id = cluster_id
def configuration(self):
return self.ng_configuration
def storage_paths(self):
return ['/mnt']
class FakeCluster(object):
def __init__(self, name, tenant, plugin, version, node_groups):
self.name = name
self.tenant = tenant
self.plugin = plugin
self.version = version
self.node_groups = node_groups
class FakeInst(object):
def __init__(self, inst_name, inst_id):
self.instance_name = inst_name
self.instance_id = inst_id
def hostname(self):
return self.instance_name
ms_inst = FakeInst('inst1', 'id1')
wk_inst = FakeInst('inst2', 'id2')
conf1 = {
"MapReduce": {},
"HDFS": {}
}
conf2 = {
"MapReduce": {},
"HDFS": {
"spam": "eggs"
}
}
ng1 = FakeNG('master', 'fl1', ['namenode', 'jobtracker'], 1,
[ms_inst], conf1, 'id1')
ng2 = FakeNG('worker', 'fl1', ['datanode', 'tasktracker'], 1,
[wk_inst], conf2, 'id1')
cluster = FakeCluster('cl1', 'ten1', 'vanilla', '1.2.1', [ng1, ng2])
cond_get_cluster.return_value = cluster
self.assertEqual(
c_h.get_config_value('HDFS', 'fs.default.name', cluster),
'hdfs://inst1:8020')
self.assertEqual(
c_h.get_config_value('HDFS', 'dfs.data.dir', node_group=ng1),
'/mnt/lib/hadoop/hdfs/datanode')
self.assertEqual(
c_h.get_config_value('HDFS', 'spam', node_group=ng2), 'eggs')
self.assertEqual(
c_h.get_config_value('HDFS', 'dfs.safemode.extension'), 30000)
self.assertRaises(RuntimeError,
c_h.get_config_value,
'MapReduce', 'spam', cluster, ng1)
