def generate_xml_configs(configs, storage_path, nn_hostname, hadoop_port): if hadoop_port is None: hadoop_port = 8020 cfg = { "fs.defaultFS": "hdfs://%s:%s" % (nn_hostname, str(hadoop_port)), "dfs.namenode.name.dir": extract_hadoop_path(storage_path, "/dfs/nn"), "dfs.datanode.data.dir": extract_hadoop_path(storage_path, "/dfs/dn"), "hadoop.tmp.dir": extract_hadoop_path(storage_path, "/dfs"), } # inserting user-defined configs for key, value in extract_hadoop_xml_confs(configs): cfg[key] = value # invoking applied configs to appropriate xml files core_all = CORE_DEFAULT if CONF.enable_data_locality: cfg.update(topology.TOPOLOGY_CONFIG) # applying vm awareness configs core_all += topology.vm_awareness_core_config() xml_configs = {"core-site": x.create_hadoop_xml(cfg, core_all), "hdfs-site": x.create_hadoop_xml(cfg, HDFS_DEFAULT)} return xml_configs
def generate_xml_configs(configs, storage_path, nn_hostname, hadoop_port): if hadoop_port is None: hadoop_port = 8020 cfg = { 'fs.defaultFS': 'hdfs://%s:%s' % (nn_hostname, str(hadoop_port)), 'dfs.namenode.name.dir': extract_hadoop_path(storage_path, '/dfs/nn'), 'dfs.datanode.data.dir': extract_hadoop_path(storage_path, '/dfs/dn'), 'hadoop.tmp.dir': extract_hadoop_path(storage_path, '/dfs'), } # inserting user-defined configs for key, value in extract_hadoop_xml_confs(configs): cfg[key] = value # invoking applied configs to appropriate xml files core_all = CORE_DEFAULT if CONF.enable_data_locality: cfg.update(topology.TOPOLOGY_CONFIG) # applying vm awareness configs core_all += topology.vm_awareness_core_config() xml_configs = { 'core-site': x.create_hadoop_xml(cfg, core_all), 'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT) } return xml_configs
def _get_core_site_node_aware_props(self): result = topo.vm_awareness_core_config() result = {c['name']: c['value'] for c in result} result.update({ 'net.topology.script.file.name': '/opt/mapr/topology.sh', 'net.topology.script.number.args': '75', }) return result
def _get_core_site_props(self, context): result = {} if context.is_node_aware: for conf in topo.vm_awareness_core_config(): result[conf['name']] = conf['value'] for conf in swift_helper.get_swift_configs(): result[conf['name']] = conf['value'] for conf in self._get_impersonation_props(): result[conf['name']] = conf['value'] return result
def generate_xml_configs(cluster, node_group, hive_mysql_passwd): oozie_hostname = vu.get_instance_hostname(vu.get_oozie(cluster)) hive_hostname = vu.get_instance_hostname(vu.get_hiveserver(cluster)) ng_configs = node_group.configuration() general_cfg = get_general_configs(hive_hostname, hive_mysql_passwd) all_cfg = generate_sahara_configs(cluster, node_group) # inserting user-defined configs for key, value in extract_xml_confs(ng_configs): all_cfg[key] = value # applying swift configs if user enabled it swift_xml_confs = swift.get_swift_configs() all_cfg = generate_cfg_from_general(all_cfg, ng_configs, general_cfg) # invoking applied configs to appropriate xml files core_all = CORE_DEFAULT + swift_xml_confs mapred_all = MAPRED_DEFAULT if CONF.enable_data_locality: all_cfg.update(topology.TOPOLOGY_CONFIG) # applying vm awareness configs core_all += topology.vm_awareness_core_config() mapred_all += topology.vm_awareness_mapred_config() xml_configs = { 'core-site': x.create_hadoop_xml(all_cfg, core_all), 'mapred-site': x.create_hadoop_xml(all_cfg, mapred_all), 'hdfs-site': x.create_hadoop_xml(all_cfg, HDFS_DEFAULT) } if hive_hostname: cfg = all_cfg cfg_filter = HIVE_DEFAULT proxy_configs = cluster.cluster_configs.get('proxy_configs') if CONF.use_identity_api_v3 and proxy_configs: cfg, cfg_filter = _inject_swift_trust_info(cfg, cfg_filter, proxy_configs) xml_configs.update({'hive-site': x.create_hadoop_xml(cfg, cfg_filter)}) LOG.debug('Generated hive-site.xml for hive {host}'.format( host=hive_hostname)) if oozie_hostname: xml_configs.update({'oozie-site': x.create_hadoop_xml(all_cfg, o_h.OOZIE_DEFAULT)}) LOG.debug('Generated oozie-site.xml for oozie {host}'.format( host=oozie_hostname)) return xml_configs
def generate_xml_configs(cluster, node_group, hive_mysql_passwd): oozie_hostname = vu.get_instance_hostname(vu.get_oozie(cluster)) hive_hostname = vu.get_instance_hostname(vu.get_hiveserver(cluster)) ng_configs = node_group.configuration() general_cfg = get_general_configs(hive_hostname, hive_mysql_passwd) all_cfg = generate_sahara_configs(cluster, node_group) # inserting user-defined configs for key, value in extract_xml_confs(ng_configs): all_cfg[key] = value # applying swift configs if user enabled it swift_xml_confs = swift.get_swift_configs() all_cfg = generate_cfg_from_general(all_cfg, ng_configs, general_cfg) # invoking applied configs to appropriate xml files core_all = CORE_DEFAULT + swift_xml_confs mapred_all = MAPRED_DEFAULT if CONF.enable_data_locality: all_cfg.update(topology.TOPOLOGY_CONFIG) # applying vm awareness configs core_all += topology.vm_awareness_core_config() mapred_all += topology.vm_awareness_mapred_config() xml_configs = { 'core-site': x.create_hadoop_xml(all_cfg, core_all), 'mapred-site': x.create_hadoop_xml(all_cfg, mapred_all), 'hdfs-site': x.create_hadoop_xml(all_cfg, HDFS_DEFAULT) } if hive_hostname: cfg = all_cfg cfg_filter = HIVE_DEFAULT proxy_configs = cluster.cluster_configs.get('proxy_configs') if CONF.use_identity_api_v3 and proxy_configs: cfg, cfg_filter = _inject_swift_trust_info(cfg, cfg_filter, proxy_configs) xml_configs.update({'hive-site': x.create_hadoop_xml(cfg, cfg_filter)}) LOG.debug('Generated hive-site.xml for hive {host}'.format( host=hive_hostname)) if oozie_hostname: xml_configs.update( {'oozie-site': x.create_hadoop_xml(all_cfg, o_h.OOZIE_DEFAULT)}) LOG.debug('Generated oozie-site.xml for oozie {host}'.format( host=oozie_hostname)) return xml_configs
def finalize_configuration(self, cluster_spec): nn_hosts = cluster_spec.determine_component_hosts('NAMENODE') if nn_hosts: props = { 'core-site': ['fs.default.name'], 'hdfs-site': ['dfs.http.address', 'dfs.https.address'] } self._replace_config_token(cluster_spec, '%NN_HOST%', nn_hosts.pop().fqdn(), props) snn_hosts = cluster_spec.determine_component_hosts( 'SECONDARY_NAMENODE') if snn_hosts: props = {'hdfs-site': ['dfs.secondary.http.address']} self._replace_config_token(cluster_spec, '%SNN_HOST%', snn_hosts.pop().fqdn(), props) # add swift properties to configuration core_site_config = cluster_spec.configurations['core-site'] for prop in self._get_swift_properties(): core_site_config[prop['name']] = prop['value'] # add topology properties to configuration, if enabled if CONF.enable_data_locality: for prop in th.vm_awareness_core_config(): core_site_config[prop['name']] = prop['value'] core_site_config.update(TOPOLOGY_CONFIG) # process storage paths to accommodate ephemeral or cinder storage nn_ng = cluster_spec.get_node_groups_containing_component( 'NAMENODE')[0] dn_node_groups = cluster_spec.get_node_groups_containing_component( 'DATANODE') common_paths = [] if dn_node_groups: common_paths = self._get_common_paths(dn_node_groups) hdfs_site_config = cluster_spec.configurations['hdfs-site'] global_config = cluster_spec.configurations['global'] hdfs_site_config['dfs.name.dir'] = self._generate_storage_path( nn_ng.storage_paths(), '/hadoop/hdfs/namenode') global_config['dfs_name_dir'] = self._generate_storage_path( nn_ng.storage_paths(), '/hadoop/hdfs/namenode') if common_paths: hdfs_site_config['dfs.data.dir'] = self._generate_storage_path( common_paths, '/hadoop/hdfs/data') global_config['dfs_data_dir'] = self._generate_storage_path( common_paths, '/hadoop/hdfs/data')
def test_core_config(self): result = th.vm_awareness_core_config() self.assertEqual(4, len(result)) for item in result: del item['description'] className = 'org.apache.hadoop.net.NetworkTopologyWithNodeGroup' self.assertIn({'name': "net.topology.impl", 'value': className}, result) self.assertIn({'name': "net.topology.nodegroup.aware", 'value': 'true'}, result) className = ('org.apache.hadoop.hdfs.server.namenode.' 'BlockPlacementPolicyWithNodeGroup') self.assertIn({'name': "dfs.block.replicator.classname", 'value': className}, result)
def generate_xml_configs(cluster, node_group, hive_mysql_passwd): oozie_hostname = _get_hostname(utils.get_oozie(cluster)) hive_hostname = _get_hostname(utils.get_hiveserver(cluster)) ng_configs = node_group.configuration() general_cfg = get_general_configs(hive_hostname, hive_mysql_passwd) all_cfg = generate_sahara_configs(cluster, node_group) # inserting user-defined configs for key, value in extract_xml_confs(ng_configs): all_cfg[key] = value # applying swift configs if user enabled it swift_xml_confs = swift.get_swift_configs() all_cfg = generate_cfg_from_general(all_cfg, ng_configs, general_cfg) # invoking applied configs to appropriate xml files core_all = CORE_DEFAULT + swift_xml_confs mapred_all = MAPRED_DEFAULT if CONF.enable_data_locality: all_cfg.update(topology.TOPOLOGY_CONFIG) # applying vm awareness configs core_all += topology.vm_awareness_core_config() mapred_all += topology.vm_awareness_mapred_config() xml_configs = { 'core-site': x.create_hadoop_xml(all_cfg, core_all), 'mapred-site': x.create_hadoop_xml(all_cfg, mapred_all), 'hdfs-site': x.create_hadoop_xml(all_cfg, HDFS_DEFAULT) } if hive_hostname: xml_configs.update( {'hive-site': x.create_hadoop_xml(all_cfg, HIVE_DEFAULT)}) LOG.debug('Generated hive-site.xml for hive % s', hive_hostname) if oozie_hostname: xml_configs.update( {'oozie-site': x.create_hadoop_xml(all_cfg, o_h.OOZIE_DEFAULT)}) LOG.debug('Generated oozie-site.xml for oozie % s', oozie_hostname) return xml_configs
def generate_xml_configs(cluster, node_group, hive_mysql_passwd): oozie_hostname = _get_hostname(utils.get_oozie(cluster)) hive_hostname = _get_hostname(utils.get_hiveserver(cluster)) ng_configs = node_group.configuration() general_cfg = get_general_configs(hive_hostname, hive_mysql_passwd) all_cfg = generate_sahara_configs(cluster, node_group) # inserting user-defined configs for key, value in extract_xml_confs(ng_configs): all_cfg[key] = value # applying swift configs if user enabled it swift_xml_confs = swift.get_swift_configs() all_cfg = generate_cfg_from_general(all_cfg, ng_configs, general_cfg) # invoking applied configs to appropriate xml files core_all = CORE_DEFAULT + swift_xml_confs mapred_all = MAPRED_DEFAULT if CONF.enable_data_locality: all_cfg.update(topology.TOPOLOGY_CONFIG) # applying vm awareness configs core_all += topology.vm_awareness_core_config() mapred_all += topology.vm_awareness_mapred_config() xml_configs = { 'core-site': x.create_hadoop_xml(all_cfg, core_all), 'mapred-site': x.create_hadoop_xml(all_cfg, mapred_all), 'hdfs-site': x.create_hadoop_xml(all_cfg, HDFS_DEFAULT) } if hive_hostname: xml_configs.update({'hive-site': x.create_hadoop_xml(all_cfg, HIVE_DEFAULT)}) LOG.debug('Generated hive-site.xml for hive % s', hive_hostname) if oozie_hostname: xml_configs.update({'oozie-site': x.create_hadoop_xml(all_cfg, o_h.OOZIE_DEFAULT)}) LOG.debug('Generated oozie-site.xml for oozie % s', oozie_hostname) return xml_configs
def finalize_configuration(self, cluster_spec): nn_hosts = cluster_spec.determine_component_hosts('NAMENODE') if nn_hosts: props = {'core-site': ['fs.default.name'], 'hdfs-site': ['dfs.http.address', 'dfs.https.address']} self._replace_config_token( cluster_spec, '%NN_HOST%', nn_hosts.pop().fqdn(), props) snn_hosts = cluster_spec.determine_component_hosts( 'SECONDARY_NAMENODE') if snn_hosts: props = {'hdfs-site': ['dfs.secondary.http.address']} self._replace_config_token( cluster_spec, '%SNN_HOST%', snn_hosts.pop().fqdn(), props) # add swift properties to configuration core_site_config = cluster_spec.configurations['core-site'] for prop in self._get_swift_properties(): core_site_config[prop['name']] = prop['value'] # add topology properties to configuration, if enabled if CONF.enable_data_locality: for prop in th.vm_awareness_core_config(): core_site_config[prop['name']] = prop['value'] core_site_config.update(TOPOLOGY_CONFIG) # process storage paths to accommodate ephemeral or cinder storage nn_ng = cluster_spec.get_node_groups_containing_component( 'NAMENODE')[0] dn_node_groups = cluster_spec.get_node_groups_containing_component( 'DATANODE') common_paths = [] if dn_node_groups: common_paths = self._get_common_paths(dn_node_groups) hdfs_site_config = cluster_spec.configurations['hdfs-site'] global_config = cluster_spec.configurations['global'] hdfs_site_config['dfs.name.dir'] = self._generate_storage_path( nn_ng.storage_paths(), '/hadoop/hdfs/namenode') global_config['dfs_name_dir'] = self._generate_storage_path( nn_ng.storage_paths(), '/hadoop/hdfs/namenode') if common_paths: hdfs_site_config['dfs.data.dir'] = self._generate_storage_path( common_paths, '/hadoop/hdfs/data') global_config['dfs_data_dir'] = self._generate_storage_path( common_paths, '/hadoop/hdfs/data')
def generate_xml_configs(configs, storage_path, nn_hostname, hadoop_port): if hadoop_port is None: hadoop_port = 8020 cfg = { 'fs.defaultFS': 'hdfs://%s:%s' % (nn_hostname, str(hadoop_port)), 'dfs.namenode.name.dir': extract_hadoop_path(storage_path, '/dfs/nn'), 'dfs.datanode.data.dir': extract_hadoop_path(storage_path, '/dfs/dn'), 'hadoop.tmp.dir': extract_hadoop_path(storage_path, '/dfs'), 'dfs.hosts': '/etc/hadoop/dn.incl', 'dfs.hosts.exclude': '/etc/hadoop/dn.excl' } # inserting user-defined configs for key, value in extract_hadoop_xml_confs(configs): cfg[key] = value # Add the swift defaults if they have not been set by the user swft_def = [] if is_swift_enabled(configs): swft_def = SWIFT_DEFAULTS swift_configs = extract_name_values(swift.get_swift_configs()) for key, value in six.iteritems(swift_configs): if key not in cfg: cfg[key] = value # invoking applied configs to appropriate xml files core_all = CORE_DEFAULT + swft_def if CONF.enable_data_locality: cfg.update(topology.TOPOLOGY_CONFIG) # applying vm awareness configs core_all += topology.vm_awareness_core_config() xml_configs = { 'core-site': x.create_hadoop_xml(cfg, core_all), 'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT) } return xml_configs
def generate_xml_configs(configs, storage_path, nn_hostname, hadoop_port): """dfs.name.dir': extract_hadoop_path(storage_path, '/lib/hadoop/hdfs/namenode'), 'dfs.data.dir': extract_hadoop_path(storage_path, '/lib/hadoop/hdfs/datanode'), 'dfs.name.dir': storage_path + 'hdfs/name', 'dfs.data.dir': storage_path + 'hdfs/data', 'dfs.hosts': '/etc/hadoop/dn.incl', 'dfs.hosts.exclude': '/etc/hadoop/dn.excl', """ if hadoop_port is None: hadoop_port = 8020 cfg = { 'fs.defaultFS': 'hdfs://%s:%s' % (nn_hostname, str(hadoop_port)), 'dfs.namenode.name.dir': extract_hadoop_path(storage_path, '/dfs/nn'), 'dfs.datanode.data.dir': extract_hadoop_path(storage_path, '/dfs/dn'), 'hadoop.tmp.dir': extract_hadoop_path(storage_path, '/dfs'), } # inserting user-defined configs for key, value in extract_hadoop_xml_confs(configs): cfg[key] = value # invoking applied configs to appropriate xml files core_all = CORE_DEFAULT if CONF.enable_data_locality: cfg.update(topology.TOPOLOGY_CONFIG) # applying vm awareness configs core_all += topology.vm_awareness_core_config() xml_configs = { 'core-site': x.create_hadoop_xml(cfg, core_all), 'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT) } return xml_configs
def vm_awareness_core_config(**kwargs): return t_helper.vm_awareness_core_config()