def _extract_configs_to_extra(self, cluster):
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
oozie = utils.get_oozie(cluster)
hive = utils.get_hiveserver(cluster)
extra = dict()
if hive:
extra['hive_mysql_passwd'] = uuidutils.generate_uuid()
for ng in cluster.node_groups:
extra[ng.id] = {
'xml':
c_helper.generate_xml_configs(
ng.configuration, ng.storage_paths, nn.hostname,
jt.hostname if jt else None,
oozie.hostname if oozie else None,
hive.hostname if hive else None,
extra['hive_mysql_passwd'] if hive else None),
'setup_script':
c_helper.generate_setup_script(
ng.storage_paths,
c_helper.extract_environment_confs(ng.configuration),
append_oozie=(oozie is not None
and oozie.node_group.id == ng.id))
}
if c_helper.is_data_locality_enabled(cluster):
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
extra['topology_data'] = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
return extra
def _extract_configs_to_extra(self, cluster):
oozie = utils.get_oozie(cluster)
hive = utils.get_hiveserver(cluster)
extra = dict()
if hive:
extra['hive_mysql_passwd'] = six.text_type(uuid.uuid4())
for ng in cluster.node_groups:
extra[ng.id] = {
'xml':
c_helper.generate_xml_configs(
cluster, ng, extra['hive_mysql_passwd'] if hive else None),
'setup_script':
c_helper.generate_setup_script(
ng.storage_paths(),
c_helper.extract_environment_confs(ng.configuration()),
append_oozie=(oozie and oozie.node_group.id == ng.id))
}
if c_helper.is_data_locality_enabled(cluster):
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
extra['topology_data'] = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
return extra
def _extract_configs_to_extra(self, cluster):
oozie = utils.get_oozie(cluster)
hive = utils.get_hiveserver(cluster)
extra = dict()
if hive:
extra['hive_mysql_passwd'] = six.text_type(uuid.uuid4())
for ng in cluster.node_groups:
extra[ng.id] = {
'xml': c_helper.generate_xml_configs(
cluster, ng, extra['hive_mysql_passwd'] if hive else None),
'setup_script': c_helper.generate_setup_script(
ng.storage_paths(),
c_helper.extract_environment_confs(ng.configuration()),
append_oozie=(
oozie and oozie.node_group.id == ng.id)
)
}
if c_helper.is_data_locality_enabled(cluster):
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
extra['topology_data'] = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
return extra
def _configure_topology_for_cluster(self, cluster, servers):
if CONF.enable_data_locality:
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
topology_str = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
for server in servers:
server.configure_topology(topology_str)
def _configure_topology_for_cluster(self, cluster, servers):
if CONF.enable_data_locality:
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
topology_str = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
for server in servers:
server.configure_topology(topology_str)
def _extract_configs_to_extra(self, cluster):
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
oozie = utils.get_oozie(cluster)
hive = utils.get_hiveserver(cluster)
extra = dict()
if hive:
extra['hive_mysql_passwd'] = uuidutils.generate_uuid()
for ng in cluster.node_groups:
extra[ng.id] = {
'xml': c_helper.generate_xml_configs(
ng.configuration,
ng.storage_paths,
nn.hostname,
jt.hostname if jt else None,
oozie.hostname if oozie else None,
hive.hostname if hive else None,
extra['hive_mysql_passwd'] if hive else None),
'setup_script': c_helper.generate_setup_script(
ng.storage_paths,
c_helper.extract_environment_confs(ng.configuration),
append_oozie=(
oozie is not None and oozie.node_group.id == ng.id)
)
}
if c_helper.is_data_locality_enabled(cluster):
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
extra['topology_data'] = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
return extra
def test_get_topology(self, swift_topology, compute_topology, novaclient):
nova = mock.Mock()
novaclient.return_value = nova
r1 = mock.Mock()
r1.hostId = "o1"
r2 = mock.Mock()
r2.hostId = "o1"
r3 = mock.Mock()
r3.hostId = "o2"
nova.servers.get.side_effect = [r1, r2, r3, r1, r2, r3]
swift_topology.return_value = {"s1": "/r1"}
compute_topology.return_value = {"o1": "/r1", "o2": "/r2"}
i1 = o.Instance()
i1.instance_id = "i1"
i1.instance_name = "i1"
i1.internal_ip = "0.0.1.1"
i1.management_ip = "1.1.1.1"
i2 = o.Instance()
i2.instance_id = "i2"
i2.instance_name = "i2"
i2.management_ip = "1.1.1.2"
i2.internal_ip = "0.0.1.2"
i3 = o.Instance()
i3.instance_id = "i3"
i3.instance_name = "i3"
i3.internal_ip = "1.1.1.3"
i3.management_ip = "0.0.1.3"
ng1 = o.NodeGroup()
ng1.name = "1"
ng1.instances = [i1, i2]
ng2 = o.NodeGroup()
ng2.name = "2"
ng2.instances = [i3]
cluster = o.Cluster
cluster.node_groups = [ng1, ng2]
top = th.generate_topology_map(cluster, False)
self.assertEqual(
top, {
"i1": "/r1",
"1.1.1.1": "/r1",
"0.0.1.1": "/r1",
"i2": "/r1",
"1.1.1.2": "/r1",
"0.0.1.2": "/r1",
"i3": "/r2",
"1.1.1.3": "/r2",
"0.0.1.3": "/r2",
"s1": "/r1"
})
top = th.generate_topology_map(cluster, True)
self.assertEqual(
top, {
"i1": "/r1/o1",
"1.1.1.1": "/r1/o1",
"0.0.1.1": "/r1/o1",
"i2": "/r1/o1",
"1.1.1.2": "/r1/o1",
"0.0.1.2": "/r1/o1",
"i3": "/r2/o2",
"1.1.1.3": "/r2/o2",
"0.0.1.3": "/r2/o2",
"s1": "/r1"
})
def test_get_topology(self,
swift_topology,
compute_topology,
novaclient):
nova = mock.Mock()
novaclient.return_value = nova
r1 = mock.Mock()
r1.hostId = "o1"
r2 = mock.Mock()
r2.hostId = "o1"
r3 = mock.Mock()
r3.hostId = "o2"
nova.servers.get.side_effect = [r1, r2, r3, r1, r2, r3]
swift_topology.return_value = {"s1": "/r1"}
compute_topology.return_value = {"o1": "/r1", "o2": "/r2"}
i1 = o.Instance()
i1.instance_id = "i1"
i1.instance_name = "i1"
i1.internal_ip = "0.0.1.1"
i1.management_ip = "1.1.1.1"
i2 = o.Instance()
i2.instance_id = "i2"
i2.instance_name = "i2"
i2.management_ip = "1.1.1.2"
i2.internal_ip = "0.0.1.2"
i3 = o.Instance()
i3.instance_id = "i3"
i3.instance_name = "i3"
i3.internal_ip = "1.1.1.3"
i3.management_ip = "0.0.1.3"
ng1 = o.NodeGroup()
ng1.name = "1"
ng1.instances = [i1, i2]
ng2 = o.NodeGroup()
ng2.name = "2"
ng2.instances = [i3]
cluster = o.Cluster
cluster.node_groups = [ng1, ng2]
top = th.generate_topology_map(cluster, False)
self.assertEqual(top, {
"i1": "/r1",
"1.1.1.1": "/r1",
"0.0.1.1": "/r1",
"i2": "/r1",
"1.1.1.2": "/r1",
"0.0.1.2": "/r1",
"i3": "/r2",
"1.1.1.3": "/r2",
"0.0.1.3": "/r2",
"s1": "/r1"
})
top = th.generate_topology_map(cluster, True)
self.assertEqual(top, {
"i1": "/r1/o1",
"1.1.1.1": "/r1/o1",
"0.0.1.1": "/r1/o1",
"i2": "/r1/o1",
"1.1.1.2": "/r1/o1",
"0.0.1.2": "/r1/o1",
"i3": "/r2/o2",
"1.1.1.3": "/r2/o2",
"0.0.1.3": "/r2/o2",
"s1": "/r1"
})
