def _extract_configs_to_extra(self, cluster): nn = utils.get_namenode(cluster) jt = utils.get_jobtracker(cluster) oozie = utils.get_oozie(cluster) hive = utils.get_hiveserver(cluster) extra = dict() if hive: extra['hive_mysql_passwd'] = uuidutils.generate_uuid() for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs( ng.configuration, ng.storage_paths, nn.hostname, jt.hostname if jt else None, oozie.hostname if oozie else None, hive.hostname if hive else None, extra['hive_mysql_passwd'] if hive else None), 'setup_script': c_helper.generate_setup_script( ng.storage_paths, c_helper.extract_environment_confs(ng.configuration), append_oozie=(oozie is not None and oozie.node_group.id == ng.id)) } if c_helper.is_data_locality_enabled(cluster): topology_data = th.generate_topology_map( cluster, CONF.enable_hypervisor_awareness) extra['topology_data'] = "\n".join( [k + " " + v for k, v in topology_data.items()]) + "\n" return extra
def _extract_configs_to_extra(self, cluster): oozie = utils.get_oozie(cluster) hive = utils.get_hiveserver(cluster) extra = dict() if hive: extra['hive_mysql_passwd'] = six.text_type(uuid.uuid4()) for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs( cluster, ng, extra['hive_mysql_passwd'] if hive else None), 'setup_script': c_helper.generate_setup_script( ng.storage_paths(), c_helper.extract_environment_confs(ng.configuration()), append_oozie=( oozie and oozie.node_group.id == ng.id) ) } if c_helper.is_data_locality_enabled(cluster): topology_data = th.generate_topology_map( cluster, CONF.enable_hypervisor_awareness) extra['topology_data'] = "\n".join( [k + " " + v for k, v in topology_data.items()]) + "\n" return extra
def _extract_configs_to_extra(self, cluster): nn = utils.get_namenode(cluster) jt = utils.get_jobtracker(cluster) oozie = utils.get_oozie(cluster) hive = utils.get_hiveserver(cluster) extra = dict() for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs(ng.configuration, ng.storage_paths, nn.hostname, jt.hostname if jt else None, oozie.hostname if oozie else None, hive.hostname if hive else None), 'setup_script': c_helper.generate_setup_script( ng.storage_paths, c_helper.extract_environment_confs(ng.configuration), append_oozie=( oozie is not None and oozie.node_group.id == ng.id) ) } return extra
def test_extract_environment_configs(self): env_configs = { "MapReduce": {"Job Tracker Heap Size": 1000, "Task Tracker Heap Size": "2000"}, "HDFS": {"Name Node Heap Size": 3000, "Data Node Heap Size": "4000"}, "Wrong-applicable-target": {"t1": 4}, } self.assertListEqual( c_h.extract_environment_confs(env_configs), [ 'HADOOP_NAMENODE_OPTS=\\"-Xmx3000m\\"', 'HADOOP_DATANODE_OPTS=\\"-Xmx4000m\\"', 'HADOOP_JOBTRACKER_OPTS=\\"-Xmx1000m\\"', 'HADOOP_TASKTRACKER_OPTS=\\"-Xmx2000m\\"', ], )
def _extract_configs(self, cluster): nn = utils.get_namenode(cluster) jt = utils.get_jobtracker(cluster) for ng in cluster.node_groups: ng.extra = { 'xml': c_helper.generate_xml_configs(ng.configuration, ng.storage_paths, nn.hostname, jt.hostname if jt else None), 'setup_script': c_helper.generate_setup_script( ng.storage_paths, c_helper.extract_environment_confs(ng.configuration) ) }
def _extract_configs_to_extra(self, cluster): nn = utils.get_namenode(cluster) jt = utils.get_jobtracker(cluster) oozie = utils.get_oozie(cluster) extra = dict() for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs( ng.configuration, ng.storage_paths, nn.hostname, jt.hostname if jt else None, oozie.hostname if oozie else None), 'setup_script': c_helper.generate_setup_script( ng.storage_paths, c_helper.extract_environment_confs(ng.configuration)) } return extra
def test_extract_environment_configs(self): env_configs = { "MapReduce": { 'Job Tracker Heap Size': 1000, 'Task Tracker Heap Size': "2000" }, "HDFS": { 'Name Node Heap Size': 3000, 'Data Node Heap Size': "4000" }, "Wrong-applicable-target": { 't1': 4 } } self.assertListEqual(c_h.extract_environment_confs(env_configs), [ 'HADOOP_NAMENODE_OPTS=\\"-Xmx3000m\\"', 'HADOOP_DATANODE_OPTS=\\"-Xmx4000m\\"', 'HADOOP_JOBTRACKER_OPTS=\\"-Xmx1000m\\"', 'HADOOP_TASKTRACKER_OPTS=\\"-Xmx2000m\\"' ])
def _extract_configs_to_extra(self, cluster): nn = utils.get_namenode(cluster) jt = utils.get_jobtracker(cluster) oozies_hostnames = [o.hostname for o in utils.get_oozies(cluster)] extra = dict() for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs(ng.configuration, ng.storage_paths, nn.hostname, jt.hostname if jt else None, oozies_hostnames), 'setup_script': c_helper.generate_setup_script( ng.storage_paths, c_helper.extract_environment_confs(ng.configuration) ) } return extra
def test_extract_environment_configs(self): env_configs = { "JobFlow": { 'Oozie Heap Size': 4000 }, "MapReduce": { 'Job Tracker Heap Size': 1000, 'Task Tracker Heap Size': "2000" }, "HDFS": { 'Name Node Heap Size': 3000, 'Data Node Heap Size': "4000" }, "Wrong-applicable-target": { 't1': 4 }} self.assertListEqual(c_h.extract_environment_confs(env_configs), ['HADOOP_NAMENODE_OPTS=\\"-Xmx3000m\\"', 'HADOOP_DATANODE_OPTS=\\"-Xmx4000m\\"', 'CATALINA_OPTS=\\"-Xmx4000m\\"', 'HADOOP_JOBTRACKER_OPTS=\\"-Xmx1000m\\"', 'HADOOP_TASKTRACKER_OPTS=\\"-Xmx2000m\\"'])
def _extract_configs_to_extra(self, cluster): nn = utils.get_namenode(cluster) jt = utils.get_jobtracker(cluster) oozie = utils.get_oozie(cluster) hive = utils.get_hiveserver(cluster) extra = dict() if hive: extra['hive_mysql_passwd'] = uuidutils.generate_uuid() for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs( ng.configuration, ng.storage_paths, nn.hostname, jt.hostname if jt else None, oozie.hostname if oozie else None, hive.hostname if hive else None, extra['hive_mysql_passwd'] if hive else None), 'setup_script': c_helper.generate_setup_script( ng.storage_paths, c_helper.extract_environment_confs(ng.configuration), append_oozie=( oozie is not None and oozie.node_group.id == ng.id) ) } if c_helper.is_data_locality_enabled(cluster): topology_data = th.generate_topology_map( cluster, CONF.enable_hypervisor_awareness) extra['topology_data'] = "\n".join( [k + " " + v for k, v in topology_data.items()]) + "\n" return extra