def generate_xml_configs(configs, storage_path, nn_hostname, hadoop_port):
if hadoop_port is None:
hadoop_port = 8020
cfg = {
"fs.defaultFS": "hdfs://%s:%s" % (nn_hostname, str(hadoop_port)),
"dfs.namenode.name.dir": extract_hadoop_path(storage_path, "/dfs/nn"),
"dfs.datanode.data.dir": extract_hadoop_path(storage_path, "/dfs/dn"),
"hadoop.tmp.dir": extract_hadoop_path(storage_path, "/dfs"),
}
# inserting user-defined configs
for key, value in extract_hadoop_xml_confs(configs):
cfg[key] = value
# invoking applied configs to appropriate xml files
core_all = CORE_DEFAULT
if CONF.enable_data_locality:
cfg.update(topology.TOPOLOGY_CONFIG)
# applying vm awareness configs
core_all += topology.vm_awareness_core_config()
xml_configs = {"core-site": x.create_hadoop_xml(cfg, core_all), "hdfs-site": x.create_hadoop_xml(cfg, HDFS_DEFAULT)}
return xml_configs
def generate_xml_configs(configs, storage_path, nn_hostname, hadoop_port):
if hadoop_port is None:
hadoop_port = 8020
cfg = {
'fs.defaultFS': 'hdfs://%s:%s' % (nn_hostname, str(hadoop_port)),
'dfs.namenode.name.dir': extract_hadoop_path(storage_path, '/dfs/nn'),
'dfs.datanode.data.dir': extract_hadoop_path(storage_path, '/dfs/dn'),
'hadoop.tmp.dir': extract_hadoop_path(storage_path, '/dfs'),
}
# inserting user-defined configs
for key, value in extract_hadoop_xml_confs(configs):
cfg[key] = value
# invoking applied configs to appropriate xml files
core_all = CORE_DEFAULT
if CONF.enable_data_locality:
cfg.update(topology.TOPOLOGY_CONFIG)
# applying vm awareness configs
core_all += topology.vm_awareness_core_config()
xml_configs = {
'core-site': x.create_hadoop_xml(cfg, core_all),
'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT)
}
return xml_configs
def _get_core_site_node_aware_props(self):
result = topo.vm_awareness_core_config()
result = {c['name']: c['value'] for c in result}
result.update({
'net.topology.script.file.name': '/opt/mapr/topology.sh',
'net.topology.script.number.args': '75',
})
return result
def _get_core_site_node_aware_props(self):
result = topo.vm_awareness_core_config()
result = {c['name']: c['value'] for c in result}
result.update({
'net.topology.script.file.name': '/opt/mapr/topology.sh',
'net.topology.script.number.args': '75',
})
return result
def _get_core_site_props(self, context):
result = {}
if context.is_node_aware:
for conf in topo.vm_awareness_core_config():
result[conf['name']] = conf['value']
for conf in swift_helper.get_swift_configs():
result[conf['name']] = conf['value']
for conf in self._get_impersonation_props():
result[conf['name']] = conf['value']
return result
def generate_xml_configs(cluster, node_group, hive_mysql_passwd):
oozie_hostname = vu.get_instance_hostname(vu.get_oozie(cluster))
hive_hostname = vu.get_instance_hostname(vu.get_hiveserver(cluster))
ng_configs = node_group.configuration()
general_cfg = get_general_configs(hive_hostname, hive_mysql_passwd)
all_cfg = generate_sahara_configs(cluster, node_group)
# inserting user-defined configs
for key, value in extract_xml_confs(ng_configs):
all_cfg[key] = value
# applying swift configs if user enabled it
swift_xml_confs = swift.get_swift_configs()
all_cfg = generate_cfg_from_general(all_cfg, ng_configs, general_cfg)
# invoking applied configs to appropriate xml files
core_all = CORE_DEFAULT + swift_xml_confs
mapred_all = MAPRED_DEFAULT
if CONF.enable_data_locality:
all_cfg.update(topology.TOPOLOGY_CONFIG)
# applying vm awareness configs
core_all += topology.vm_awareness_core_config()
mapred_all += topology.vm_awareness_mapred_config()
xml_configs = {
'core-site': x.create_hadoop_xml(all_cfg, core_all),
'mapred-site': x.create_hadoop_xml(all_cfg, mapred_all),
'hdfs-site': x.create_hadoop_xml(all_cfg, HDFS_DEFAULT)
}
if hive_hostname:
cfg = all_cfg
cfg_filter = HIVE_DEFAULT
proxy_configs = cluster.cluster_configs.get('proxy_configs')
if CONF.use_identity_api_v3 and proxy_configs:
cfg, cfg_filter = _inject_swift_trust_info(cfg,
cfg_filter,
proxy_configs)
xml_configs.update({'hive-site':
x.create_hadoop_xml(cfg, cfg_filter)})
LOG.debug('Generated hive-site.xml for hive {host}'.format(
host=hive_hostname))
if oozie_hostname:
xml_configs.update({'oozie-site':
x.create_hadoop_xml(all_cfg, o_h.OOZIE_DEFAULT)})
LOG.debug('Generated oozie-site.xml for oozie {host}'.format(
host=oozie_hostname))
return xml_configs
def generate_xml_configs(cluster, node_group, hive_mysql_passwd):
oozie_hostname = vu.get_instance_hostname(vu.get_oozie(cluster))
hive_hostname = vu.get_instance_hostname(vu.get_hiveserver(cluster))
ng_configs = node_group.configuration()
general_cfg = get_general_configs(hive_hostname, hive_mysql_passwd)
all_cfg = generate_sahara_configs(cluster, node_group)
# inserting user-defined configs
for key, value in extract_xml_confs(ng_configs):
all_cfg[key] = value
# applying swift configs if user enabled it
swift_xml_confs = swift.get_swift_configs()
all_cfg = generate_cfg_from_general(all_cfg, ng_configs, general_cfg)
# invoking applied configs to appropriate xml files
core_all = CORE_DEFAULT + swift_xml_confs
mapred_all = MAPRED_DEFAULT
if CONF.enable_data_locality:
all_cfg.update(topology.TOPOLOGY_CONFIG)
# applying vm awareness configs
core_all += topology.vm_awareness_core_config()
mapred_all += topology.vm_awareness_mapred_config()
xml_configs = {
'core-site': x.create_hadoop_xml(all_cfg, core_all),
'mapred-site': x.create_hadoop_xml(all_cfg, mapred_all),
'hdfs-site': x.create_hadoop_xml(all_cfg, HDFS_DEFAULT)
}
if hive_hostname:
cfg = all_cfg
cfg_filter = HIVE_DEFAULT
proxy_configs = cluster.cluster_configs.get('proxy_configs')
if CONF.use_identity_api_v3 and proxy_configs:
cfg, cfg_filter = _inject_swift_trust_info(cfg, cfg_filter,
proxy_configs)
xml_configs.update({'hive-site': x.create_hadoop_xml(cfg, cfg_filter)})
LOG.debug('Generated hive-site.xml for hive {host}'.format(
host=hive_hostname))
if oozie_hostname:
xml_configs.update(
{'oozie-site': x.create_hadoop_xml(all_cfg, o_h.OOZIE_DEFAULT)})
LOG.debug('Generated oozie-site.xml for oozie {host}'.format(
host=oozie_hostname))
return xml_configs
def finalize_configuration(self, cluster_spec):
nn_hosts = cluster_spec.determine_component_hosts('NAMENODE')
if nn_hosts:
props = {
'core-site': ['fs.default.name'],
'hdfs-site': ['dfs.http.address', 'dfs.https.address']
}
self._replace_config_token(cluster_spec, '%NN_HOST%',
nn_hosts.pop().fqdn(), props)
snn_hosts = cluster_spec.determine_component_hosts(
'SECONDARY_NAMENODE')
if snn_hosts:
props = {'hdfs-site': ['dfs.secondary.http.address']}
self._replace_config_token(cluster_spec, '%SNN_HOST%',
snn_hosts.pop().fqdn(), props)
# add swift properties to configuration
core_site_config = cluster_spec.configurations['core-site']
for prop in self._get_swift_properties():
core_site_config[prop['name']] = prop['value']
# add topology properties to configuration, if enabled
if CONF.enable_data_locality:
for prop in th.vm_awareness_core_config():
core_site_config[prop['name']] = prop['value']
core_site_config.update(TOPOLOGY_CONFIG)
# process storage paths to accommodate ephemeral or cinder storage
nn_ng = cluster_spec.get_node_groups_containing_component(
'NAMENODE')[0]
dn_node_groups = cluster_spec.get_node_groups_containing_component(
'DATANODE')
common_paths = []
if dn_node_groups:
common_paths = self._get_common_paths(dn_node_groups)
hdfs_site_config = cluster_spec.configurations['hdfs-site']
global_config = cluster_spec.configurations['global']
hdfs_site_config['dfs.name.dir'] = self._generate_storage_path(
nn_ng.storage_paths(), '/hadoop/hdfs/namenode')
global_config['dfs_name_dir'] = self._generate_storage_path(
nn_ng.storage_paths(), '/hadoop/hdfs/namenode')
if common_paths:
hdfs_site_config['dfs.data.dir'] = self._generate_storage_path(
common_paths, '/hadoop/hdfs/data')
global_config['dfs_data_dir'] = self._generate_storage_path(
common_paths, '/hadoop/hdfs/data')
def test_core_config(self):
result = th.vm_awareness_core_config()
self.assertEqual(4, len(result))
for item in result:
del item['description']
className = 'org.apache.hadoop.net.NetworkTopologyWithNodeGroup'
self.assertIn({'name': "net.topology.impl",
'value': className},
result)
self.assertIn({'name': "net.topology.nodegroup.aware",
'value': 'true'},
result)
className = ('org.apache.hadoop.hdfs.server.namenode.'
'BlockPlacementPolicyWithNodeGroup')
self.assertIn({'name': "dfs.block.replicator.classname",
'value': className},
result)
def generate_xml_configs(cluster, node_group, hive_mysql_passwd):
oozie_hostname = _get_hostname(utils.get_oozie(cluster))
hive_hostname = _get_hostname(utils.get_hiveserver(cluster))
ng_configs = node_group.configuration()
general_cfg = get_general_configs(hive_hostname, hive_mysql_passwd)
all_cfg = generate_sahara_configs(cluster, node_group)
# inserting user-defined configs
for key, value in extract_xml_confs(ng_configs):
all_cfg[key] = value
# applying swift configs if user enabled it
swift_xml_confs = swift.get_swift_configs()
all_cfg = generate_cfg_from_general(all_cfg, ng_configs, general_cfg)
# invoking applied configs to appropriate xml files
core_all = CORE_DEFAULT + swift_xml_confs
mapred_all = MAPRED_DEFAULT
if CONF.enable_data_locality:
all_cfg.update(topology.TOPOLOGY_CONFIG)
# applying vm awareness configs
core_all += topology.vm_awareness_core_config()
mapred_all += topology.vm_awareness_mapred_config()
xml_configs = {
'core-site': x.create_hadoop_xml(all_cfg, core_all),
'mapred-site': x.create_hadoop_xml(all_cfg, mapred_all),
'hdfs-site': x.create_hadoop_xml(all_cfg, HDFS_DEFAULT)
}
if hive_hostname:
xml_configs.update(
{'hive-site': x.create_hadoop_xml(all_cfg, HIVE_DEFAULT)})
LOG.debug('Generated hive-site.xml for hive % s', hive_hostname)
if oozie_hostname:
xml_configs.update(
{'oozie-site': x.create_hadoop_xml(all_cfg, o_h.OOZIE_DEFAULT)})
LOG.debug('Generated oozie-site.xml for oozie % s', oozie_hostname)
return xml_configs
def generate_xml_configs(cluster, node_group, hive_mysql_passwd):
oozie_hostname = _get_hostname(utils.get_oozie(cluster))
hive_hostname = _get_hostname(utils.get_hiveserver(cluster))
ng_configs = node_group.configuration()
general_cfg = get_general_configs(hive_hostname, hive_mysql_passwd)
all_cfg = generate_sahara_configs(cluster, node_group)
# inserting user-defined configs
for key, value in extract_xml_confs(ng_configs):
all_cfg[key] = value
# applying swift configs if user enabled it
swift_xml_confs = swift.get_swift_configs()
all_cfg = generate_cfg_from_general(all_cfg, ng_configs, general_cfg)
# invoking applied configs to appropriate xml files
core_all = CORE_DEFAULT + swift_xml_confs
mapred_all = MAPRED_DEFAULT
if CONF.enable_data_locality:
all_cfg.update(topology.TOPOLOGY_CONFIG)
# applying vm awareness configs
core_all += topology.vm_awareness_core_config()
mapred_all += topology.vm_awareness_mapred_config()
xml_configs = {
'core-site': x.create_hadoop_xml(all_cfg, core_all),
'mapred-site': x.create_hadoop_xml(all_cfg, mapred_all),
'hdfs-site': x.create_hadoop_xml(all_cfg, HDFS_DEFAULT)
}
if hive_hostname:
xml_configs.update({'hive-site':
x.create_hadoop_xml(all_cfg, HIVE_DEFAULT)})
LOG.debug('Generated hive-site.xml for hive % s', hive_hostname)
if oozie_hostname:
xml_configs.update({'oozie-site':
x.create_hadoop_xml(all_cfg, o_h.OOZIE_DEFAULT)})
LOG.debug('Generated oozie-site.xml for oozie % s', oozie_hostname)
return xml_configs
def finalize_configuration(self, cluster_spec):
nn_hosts = cluster_spec.determine_component_hosts('NAMENODE')
if nn_hosts:
props = {'core-site': ['fs.default.name'],
'hdfs-site': ['dfs.http.address', 'dfs.https.address']}
self._replace_config_token(
cluster_spec, '%NN_HOST%', nn_hosts.pop().fqdn(), props)
snn_hosts = cluster_spec.determine_component_hosts(
'SECONDARY_NAMENODE')
if snn_hosts:
props = {'hdfs-site': ['dfs.secondary.http.address']}
self._replace_config_token(
cluster_spec, '%SNN_HOST%', snn_hosts.pop().fqdn(), props)
# add swift properties to configuration
core_site_config = cluster_spec.configurations['core-site']
for prop in self._get_swift_properties():
core_site_config[prop['name']] = prop['value']
# add topology properties to configuration, if enabled
if CONF.enable_data_locality:
for prop in th.vm_awareness_core_config():
core_site_config[prop['name']] = prop['value']
core_site_config.update(TOPOLOGY_CONFIG)
# process storage paths to accommodate ephemeral or cinder storage
nn_ng = cluster_spec.get_node_groups_containing_component(
'NAMENODE')[0]
dn_node_groups = cluster_spec.get_node_groups_containing_component(
'DATANODE')
common_paths = []
if dn_node_groups:
common_paths = self._get_common_paths(dn_node_groups)
hdfs_site_config = cluster_spec.configurations['hdfs-site']
global_config = cluster_spec.configurations['global']
hdfs_site_config['dfs.name.dir'] = self._generate_storage_path(
nn_ng.storage_paths(), '/hadoop/hdfs/namenode')
global_config['dfs_name_dir'] = self._generate_storage_path(
nn_ng.storage_paths(), '/hadoop/hdfs/namenode')
if common_paths:
hdfs_site_config['dfs.data.dir'] = self._generate_storage_path(
common_paths, '/hadoop/hdfs/data')
global_config['dfs_data_dir'] = self._generate_storage_path(
common_paths, '/hadoop/hdfs/data')
def test_core_config(self):
result = th.vm_awareness_core_config()
self.assertEqual(4, len(result))
for item in result:
del item['description']
className = 'org.apache.hadoop.net.NetworkTopologyWithNodeGroup'
self.assertIn({'name': "net.topology.impl",
'value': className},
result)
self.assertIn({'name': "net.topology.nodegroup.aware",
'value': 'true'},
result)
className = ('org.apache.hadoop.hdfs.server.namenode.'
'BlockPlacementPolicyWithNodeGroup')
self.assertIn({'name': "dfs.block.replicator.classname",
'value': className},
result)
def generate_xml_configs(configs, storage_path, nn_hostname, hadoop_port):
if hadoop_port is None:
hadoop_port = 8020
cfg = {
'fs.defaultFS': 'hdfs://%s:%s' % (nn_hostname, str(hadoop_port)),
'dfs.namenode.name.dir': extract_hadoop_path(storage_path,
'/dfs/nn'),
'dfs.datanode.data.dir': extract_hadoop_path(storage_path,
'/dfs/dn'),
'hadoop.tmp.dir': extract_hadoop_path(storage_path,
'/dfs'),
'dfs.hosts': '/etc/hadoop/dn.incl',
'dfs.hosts.exclude': '/etc/hadoop/dn.excl'
}
# inserting user-defined configs
for key, value in extract_hadoop_xml_confs(configs):
cfg[key] = value
# Add the swift defaults if they have not been set by the user
swft_def = []
if is_swift_enabled(configs):
swft_def = SWIFT_DEFAULTS
swift_configs = extract_name_values(swift.get_swift_configs())
for key, value in six.iteritems(swift_configs):
if key not in cfg:
cfg[key] = value
# invoking applied configs to appropriate xml files
core_all = CORE_DEFAULT + swft_def
if CONF.enable_data_locality:
cfg.update(topology.TOPOLOGY_CONFIG)
# applying vm awareness configs
core_all += topology.vm_awareness_core_config()
xml_configs = {
'core-site': x.create_hadoop_xml(cfg, core_all),
'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT)
}
return xml_configs
def generate_xml_configs(configs, storage_path, nn_hostname, hadoop_port):
if hadoop_port is None:
hadoop_port = 8020
cfg = {
'fs.defaultFS': 'hdfs://%s:%s' % (nn_hostname, str(hadoop_port)),
'dfs.namenode.name.dir': extract_hadoop_path(storage_path,
'/dfs/nn'),
'dfs.datanode.data.dir': extract_hadoop_path(storage_path,
'/dfs/dn'),
'hadoop.tmp.dir': extract_hadoop_path(storage_path,
'/dfs'),
'dfs.hosts': '/etc/hadoop/dn.incl',
'dfs.hosts.exclude': '/etc/hadoop/dn.excl'
}
# inserting user-defined configs
for key, value in extract_hadoop_xml_confs(configs):
cfg[key] = value
# Add the swift defaults if they have not been set by the user
swft_def = []
if is_swift_enabled(configs):
swft_def = SWIFT_DEFAULTS
swift_configs = extract_name_values(swift.get_swift_configs())
for key, value in six.iteritems(swift_configs):
if key not in cfg:
cfg[key] = value
# invoking applied configs to appropriate xml files
core_all = CORE_DEFAULT + swft_def
if CONF.enable_data_locality:
cfg.update(topology.TOPOLOGY_CONFIG)
# applying vm awareness configs
core_all += topology.vm_awareness_core_config()
xml_configs = {
'core-site': x.create_hadoop_xml(cfg, core_all),
'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT)
}
return xml_configs
def generate_xml_configs(configs, storage_path, nn_hostname, hadoop_port):
"""dfs.name.dir': extract_hadoop_path(storage_path,
'/lib/hadoop/hdfs/namenode'),
'dfs.data.dir': extract_hadoop_path(storage_path,
'/lib/hadoop/hdfs/datanode'),
'dfs.name.dir': storage_path + 'hdfs/name',
'dfs.data.dir': storage_path + 'hdfs/data',
'dfs.hosts': '/etc/hadoop/dn.incl',
'dfs.hosts.exclude': '/etc/hadoop/dn.excl',
"""
if hadoop_port is None:
hadoop_port = 8020
cfg = {
'fs.defaultFS': 'hdfs://%s:%s' % (nn_hostname, str(hadoop_port)),
'dfs.namenode.name.dir': extract_hadoop_path(storage_path,
'/dfs/nn'),
'dfs.datanode.data.dir': extract_hadoop_path(storage_path,
'/dfs/dn'),
'hadoop.tmp.dir': extract_hadoop_path(storage_path,
'/dfs'),
}
# inserting user-defined configs
for key, value in extract_hadoop_xml_confs(configs):
cfg[key] = value
# invoking applied configs to appropriate xml files
core_all = CORE_DEFAULT
if CONF.enable_data_locality:
cfg.update(topology.TOPOLOGY_CONFIG)
# applying vm awareness configs
core_all += topology.vm_awareness_core_config()
xml_configs = {
'core-site': x.create_hadoop_xml(cfg, core_all),
'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT)
}
return xml_configs
def vm_awareness_core_config(**kwargs):
return t_helper.vm_awareness_core_config()
def vm_awareness_core_config(**kwargs):
return t_helper.vm_awareness_core_config()
