def _extract_configs_to_extra(self, cluster): nn = utils.get_namenode(cluster) jt = utils.get_jobtracker(cluster) oozie = utils.get_oozie(cluster) hive = utils.get_hiveserver(cluster) extra = dict() if hive: extra['hive_mysql_passwd'] = uuidutils.generate_uuid() for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs( ng.configuration, ng.storage_paths, nn.hostname, jt.hostname if jt else None, oozie.hostname if oozie else None, hive.hostname if hive else None, extra['hive_mysql_passwd'] if hive else None), 'setup_script': c_helper.generate_setup_script( ng.storage_paths, c_helper.extract_environment_confs(ng.configuration), append_oozie=(oozie is not None and oozie.node_group.id == ng.id)) } if c_helper.is_data_locality_enabled(cluster): topology_data = th.generate_topology_map( cluster, CONF.enable_hypervisor_awareness) extra['topology_data'] = "\n".join( [k + " " + v for k, v in topology_data.items()]) + "\n" return extra
def _extract_configs_to_extra(self, cluster): oozie = utils.get_oozie(cluster) hive = utils.get_hiveserver(cluster) extra = dict() if hive: extra['hive_mysql_passwd'] = six.text_type(uuid.uuid4()) for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs( cluster, ng, extra['hive_mysql_passwd'] if hive else None), 'setup_script': c_helper.generate_setup_script( ng.storage_paths(), c_helper.extract_environment_confs(ng.configuration()), append_oozie=(oozie and oozie.node_group.id == ng.id)) } if c_helper.is_data_locality_enabled(cluster): topology_data = th.generate_topology_map( cluster, CONF.enable_hypervisor_awareness) extra['topology_data'] = "\n".join( [k + " " + v for k, v in topology_data.items()]) + "\n" return extra
def _extract_configs_to_extra(self, cluster): oozie = utils.get_oozie(cluster) hive = utils.get_hiveserver(cluster) extra = dict() if hive: extra['hive_mysql_passwd'] = six.text_type(uuid.uuid4()) for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs( cluster, ng, extra['hive_mysql_passwd'] if hive else None), 'setup_script': c_helper.generate_setup_script( ng.storage_paths(), c_helper.extract_environment_confs(ng.configuration()), append_oozie=( oozie and oozie.node_group.id == ng.id) ) } if c_helper.is_data_locality_enabled(cluster): topology_data = th.generate_topology_map( cluster, CONF.enable_hypervisor_awareness) extra['topology_data'] = "\n".join( [k + " " + v for k, v in topology_data.items()]) + "\n" return extra
def _configure_topology_for_cluster(self, cluster, servers): if CONF.enable_data_locality: topology_data = th.generate_topology_map( cluster, CONF.enable_hypervisor_awareness) topology_str = "\n".join( [k + " " + v for k, v in topology_data.items()]) + "\n" for server in servers: server.configure_topology(topology_str)
def _extract_configs_to_extra(self, cluster): nn = utils.get_namenode(cluster) jt = utils.get_jobtracker(cluster) oozie = utils.get_oozie(cluster) hive = utils.get_hiveserver(cluster) extra = dict() if hive: extra['hive_mysql_passwd'] = uuidutils.generate_uuid() for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs( ng.configuration, ng.storage_paths, nn.hostname, jt.hostname if jt else None, oozie.hostname if oozie else None, hive.hostname if hive else None, extra['hive_mysql_passwd'] if hive else None), 'setup_script': c_helper.generate_setup_script( ng.storage_paths, c_helper.extract_environment_confs(ng.configuration), append_oozie=( oozie is not None and oozie.node_group.id == ng.id) ) } if c_helper.is_data_locality_enabled(cluster): topology_data = th.generate_topology_map( cluster, CONF.enable_hypervisor_awareness) extra['topology_data'] = "\n".join( [k + " " + v for k, v in topology_data.items()]) + "\n" return extra
def test_get_topology(self, swift_topology, compute_topology, novaclient): nova = mock.Mock() novaclient.return_value = nova r1 = mock.Mock() r1.hostId = "o1" r2 = mock.Mock() r2.hostId = "o1" r3 = mock.Mock() r3.hostId = "o2" nova.servers.get.side_effect = [r1, r2, r3, r1, r2, r3] swift_topology.return_value = {"s1": "/r1"} compute_topology.return_value = {"o1": "/r1", "o2": "/r2"} i1 = o.Instance() i1.instance_id = "i1" i1.instance_name = "i1" i1.internal_ip = "0.0.1.1" i1.management_ip = "1.1.1.1" i2 = o.Instance() i2.instance_id = "i2" i2.instance_name = "i2" i2.management_ip = "1.1.1.2" i2.internal_ip = "0.0.1.2" i3 = o.Instance() i3.instance_id = "i3" i3.instance_name = "i3" i3.internal_ip = "1.1.1.3" i3.management_ip = "0.0.1.3" ng1 = o.NodeGroup() ng1.name = "1" ng1.instances = [i1, i2] ng2 = o.NodeGroup() ng2.name = "2" ng2.instances = [i3] cluster = o.Cluster cluster.node_groups = [ng1, ng2] top = th.generate_topology_map(cluster, False) self.assertEqual( top, { "i1": "/r1", "1.1.1.1": "/r1", "0.0.1.1": "/r1", "i2": "/r1", "1.1.1.2": "/r1", "0.0.1.2": "/r1", "i3": "/r2", "1.1.1.3": "/r2", "0.0.1.3": "/r2", "s1": "/r1" }) top = th.generate_topology_map(cluster, True) self.assertEqual( top, { "i1": "/r1/o1", "1.1.1.1": "/r1/o1", "0.0.1.1": "/r1/o1", "i2": "/r1/o1", "1.1.1.2": "/r1/o1", "0.0.1.2": "/r1/o1", "i3": "/r2/o2", "1.1.1.3": "/r2/o2", "0.0.1.3": "/r2/o2", "s1": "/r1" })
def test_get_topology(self, swift_topology, compute_topology, novaclient): nova = mock.Mock() novaclient.return_value = nova r1 = mock.Mock() r1.hostId = "o1" r2 = mock.Mock() r2.hostId = "o1" r3 = mock.Mock() r3.hostId = "o2" nova.servers.get.side_effect = [r1, r2, r3, r1, r2, r3] swift_topology.return_value = {"s1": "/r1"} compute_topology.return_value = {"o1": "/r1", "o2": "/r2"} i1 = o.Instance() i1.instance_id = "i1" i1.instance_name = "i1" i1.internal_ip = "0.0.1.1" i1.management_ip = "1.1.1.1" i2 = o.Instance() i2.instance_id = "i2" i2.instance_name = "i2" i2.management_ip = "1.1.1.2" i2.internal_ip = "0.0.1.2" i3 = o.Instance() i3.instance_id = "i3" i3.instance_name = "i3" i3.internal_ip = "1.1.1.3" i3.management_ip = "0.0.1.3" ng1 = o.NodeGroup() ng1.name = "1" ng1.instances = [i1, i2] ng2 = o.NodeGroup() ng2.name = "2" ng2.instances = [i3] cluster = o.Cluster cluster.node_groups = [ng1, ng2] top = th.generate_topology_map(cluster, False) self.assertEqual(top, { "i1": "/r1", "1.1.1.1": "/r1", "0.0.1.1": "/r1", "i2": "/r1", "1.1.1.2": "/r1", "0.0.1.2": "/r1", "i3": "/r2", "1.1.1.3": "/r2", "0.0.1.3": "/r2", "s1": "/r1" }) top = th.generate_topology_map(cluster, True) self.assertEqual(top, { "i1": "/r1/o1", "1.1.1.1": "/r1/o1", "0.0.1.1": "/r1/o1", "i2": "/r1/o1", "1.1.1.2": "/r1/o1", "0.0.1.2": "/r1/o1", "i3": "/r2/o2", "1.1.1.3": "/r2/o2", "0.0.1.3": "/r2/o2", "s1": "/r1" })