def _configure_swift(client, cluster):
swift_enable = c_helper.get_config_value(
cluster.cluster_configs.get('general'), c_helper.ENABLE_SWIFT)
if swift_enable:
swift_configs = swift.get_swift_configs()
for conf in swift_configs:
client.params.hadoop.add(conf['name'], conf['value'])
def generate_xml_configs(configs, storage_path, nn_hostname,
jt_hostname, oozies_hostnames):
# inserting common configs depends on provisioned VMs and HDFS placement
# TODO(aignatov): should be moved to cluster context
cfg = {
'fs.default.name': 'hdfs://%s:8020' % nn_hostname,
'dfs.name.dir': extract_hadoop_path(storage_path,
'/lib/hadoop/hdfs/namenode'),
'dfs.data.dir': extract_hadoop_path(storage_path,
'/lib/hadoop/hdfs/datanode'),
'dfs.hosts': '/etc/hadoop/dn.incl',
'dfs.hosts.exclude': '/etc/hadoop/dn.excl',
}
if jt_hostname:
mr_cfg = {
'mapred.job.tracker': '%s:8021' % jt_hostname,
'mapred.system.dir': extract_hadoop_path(storage_path,
'/mapred/mapredsystem'),
'mapred.local.dir': extract_hadoop_path(storage_path,
'/lib/hadoop/mapred'),
'mapred.hosts': '/etc/hadoop/tt.incl',
'mapred.hosts.exclude': '/etc/hadoop/tt.excl',
}
cfg.update(mr_cfg)
if oozies_hostnames:
o_cfg = {
'hadoop.proxyuser.hadoop.hosts': ",".join(
["localhost"] + oozies_hostnames),
'hadoop.proxyuser.hadoop.groups': 'hadoop'
}
cfg.update(o_cfg)
LOG.debug('Applied Oozie configs for core-site.xml')
# inserting user-defined configs
for key, value in extract_xml_confs(configs):
cfg[key] = value
# applying swift configs if user enabled it
swift_xml_confs = []
#TODO(aignatov): should be changed. General configs not only Swift
swift_in_config = False
if ('general' in configs and
ENABLE_SWIFT.name in configs['general']):
swift_in_config = True
if ((swift_in_config and configs['general'][ENABLE_SWIFT.name]) or
(not swift_in_config and ENABLE_SWIFT.default_value)):
swift_xml_confs = swift.get_swift_configs()
cfg.update(extract_name_values(swift_xml_confs))
LOG.info("Swift integration is enabled")
# invoking applied configs to appropriate xml files
xml_configs = {
'core-site': x.create_hadoop_xml(cfg, CORE_DEFAULT + swift_xml_confs),
'mapred-site': x.create_hadoop_xml(cfg, MAPRED_DEFAULT),
'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT)
}
return xml_configs
def generate_xml_configs(configs, storage_path, nn_hostname, jt_hostname, oozie_hostname):
# inserting common configs depends on provisioned VMs and HDFS placement
# TODO(aignatov): should be moved to cluster context
cfg = {
"fs.default.name": "hdfs://%s:8020" % nn_hostname,
"dfs.name.dir": extract_hadoop_path(storage_path, "/lib/hadoop/hdfs/namenode"),
"dfs.data.dir": extract_hadoop_path(storage_path, "/lib/hadoop/hdfs/datanode"),
"dfs.hosts": "/etc/hadoop/dn.incl",
"dfs.hosts.exclude": "/etc/hadoop/dn.excl",
}
if jt_hostname:
mr_cfg = {
"mapred.job.tracker": "%s:8021" % jt_hostname,
"mapred.system.dir": extract_hadoop_path(storage_path, "/mapred/mapredsystem"),
"mapred.local.dir": extract_hadoop_path(storage_path, "/lib/hadoop/mapred"),
"mapred.hosts": "/etc/hadoop/tt.incl",
"mapred.hosts.exclude": "/etc/hadoop/tt.excl",
}
cfg.update(mr_cfg)
if oozie_hostname:
o_cfg = {
"hadoop.proxyuser.hadoop.hosts": "localhost," + oozie_hostname,
"hadoop.proxyuser.hadoop.groups": "hadoop",
}
cfg.update(o_cfg)
LOG.debug("Applied Oozie configs for core-site.xml")
cfg.update(o_h.get_oozie_required_xml_configs())
LOG.debug("Applied Oozie configs for oozie-site.xml")
# inserting user-defined configs
for key, value in extract_xml_confs(configs):
cfg[key] = value
# applying swift configs if user enabled it
swift_xml_confs = []
# TODO(aignatov): should be changed. General configs not only Swift
swift_in_config = False
if "general" in configs and ENABLE_SWIFT.name in configs["general"]:
swift_in_config = True
if (swift_in_config and configs["general"][ENABLE_SWIFT.name]) or (
not swift_in_config and ENABLE_SWIFT.default_value
):
swift_xml_confs = swift.get_swift_configs()
cfg.update(extract_name_values(swift_xml_confs))
LOG.info("Swift integration is enabled")
# invoking applied configs to appropriate xml files
xml_configs = {
"core-site": x.create_hadoop_xml(cfg, CORE_DEFAULT + swift_xml_confs),
"mapred-site": x.create_hadoop_xml(cfg, MAPRED_DEFAULT),
"hdfs-site": x.create_hadoop_xml(cfg, HDFS_DEFAULT),
}
if oozie_hostname:
xml_configs.update({"oozie-site": x.create_hadoop_xml(cfg, o_h.OOZIE_DEFAULT)})
LOG.debug("Generated oozie-site.xml for oozie % s", oozie_hostname)
return xml_configs
def _configure_swift(client, cluster):
swift_enable = c_helper.get_config_value(
cluster.cluster_configs.get('general'), c_helper.ENABLE_SWIFT)
if swift_enable:
swift_configs = swift.get_swift_configs()
for conf in swift_configs:
client.params.hadoop.add(conf['name'], conf['value'])
def _append_swift_inputs(self, user_inputs):
swift_props = h.get_swift_configs()
for prop in swift_props:
config = p.Config(prop['name'], 'general', 'cluster')
setattr(config, 'tag', 'core-site')
ui = p.UserInput(config, prop['value'])
user_inputs.append(ui)
def test_get_swift_configs(self, authUrlConfig):
authUrlConfig.return_value = "http://localhost:8080/v2.0/"
result = h.get_swift_configs()
self.assertEqual(7, len(result))
self.assertIn({'name': "fs.swift.service.savanna.tenant",
'value': 'test_tenant', 'description': ''}, result)
self.assertIn({'name': "fs.swift.service.savanna.http.port",
'value': '8080', 'description': ''}, result)
def test_get_swift_configs(self, authUrlConfig):
authUrlConfig.return_value = "http://localhost:8080/v2.0/tokens"
result = h.get_swift_configs()
self.assertEqual(7, len(result))
self.assertIn({'name': "fs.swift.service.savanna.location-aware",
'value': 'true', 'description': ''}, result)
self.assertIn({'name': "fs.swift.service.savanna.tenant",
'value': 'test_tenant', 'description': ''}, result)
self.assertIn({'name': "fs.swift.service.savanna.http.port",
'value': '8080', 'description': ''}, result)
def set_general_configs(hive_hostname):
GENERAL_CONFS.update({
ENABLE_SWIFT.name: {
'default_value': ENABLE_SWIFT.default_value,
'conf': extract_name_values(swift.get_swift_configs())
},
ENABLE_MYSQL.name: {
'default_value': ENABLE_MYSQL.default_value,
'conf': m_h.get_required_mysql_configs(hive_hostname)
}
})
def test_get_swift_configs(self, authUrlConfig):
main.CONF.set_override("os_auth_protocol", 'http')
main.CONF.set_override("os_auth_port", '8080')
try:
authUrlConfig.return_value = "http://localhost:8080/v2.0/"
result = h.get_swift_configs()
self.assertEqual(7, len(result))
self.assertIn({'name': "fs.swift.service.savanna.tenant",
'value': 'test_tenant', 'description': ''}, result)
self.assertIn({'name': "fs.swift.service.savanna.http.port",
'value': '8080', 'description': ''}, result)
finally:
main.CONF.clear_override("os_auth_protocol")
main.CONF.clear_override("os_auth_port")
def generate_xml_configs(cluster, node_group, hive_mysql_passwd):
oozie_hostname = _get_hostname(utils.get_oozie(cluster))
hive_hostname = _get_hostname(utils.get_hiveserver(cluster))
ng_configs = node_group.configuration()
general_cfg = get_general_configs(hive_hostname, hive_mysql_passwd)
all_cfg = generate_savanna_configs(cluster, node_group)
# inserting user-defined configs
for key, value in extract_xml_confs(ng_configs):
all_cfg[key] = value
# applying swift configs if user enabled it
swift_xml_confs = swift.get_swift_configs()
all_cfg = generate_cfg_from_general(all_cfg, ng_configs, general_cfg)
# invoking applied configs to appropriate xml files
core_all = CORE_DEFAULT + swift_xml_confs
mapred_all = MAPRED_DEFAULT
if CONF.enable_data_locality:
all_cfg.update(topology.TOPOLOGY_CONFIG)
# applying vm awareness configs
core_all += topology.vm_awareness_core_config()
mapred_all += topology.vm_awareness_mapred_config()
xml_configs = {
'core-site': x.create_hadoop_xml(all_cfg, core_all),
'mapred-site': x.create_hadoop_xml(all_cfg, mapred_all),
'hdfs-site': x.create_hadoop_xml(all_cfg, HDFS_DEFAULT)
}
if hive_hostname:
xml_configs.update(
{'hive-site': x.create_hadoop_xml(all_cfg, HIVE_DEFAULT)})
LOG.debug('Generated hive-site.xml for hive % s', hive_hostname)
if oozie_hostname:
xml_configs.update(
{'oozie-site': x.create_hadoop_xml(all_cfg, o_h.OOZIE_DEFAULT)})
LOG.debug('Generated oozie-site.xml for oozie % s', oozie_hostname)
return xml_configs
def generate_xml_configs(cluster, node_group, hive_mysql_passwd):
oozie_hostname = _get_hostname(utils.get_oozie(cluster))
hive_hostname = _get_hostname(utils.get_hiveserver(cluster))
ng_configs = node_group.configuration()
general_cfg = get_general_configs(hive_hostname, hive_mysql_passwd)
all_cfg = generate_savanna_configs(cluster, node_group)
# inserting user-defined configs
for key, value in extract_xml_confs(ng_configs):
all_cfg[key] = value
# applying swift configs if user enabled it
swift_xml_confs = swift.get_swift_configs()
all_cfg = generate_cfg_from_general(all_cfg, ng_configs, general_cfg)
# invoking applied configs to appropriate xml files
core_all = CORE_DEFAULT + swift_xml_confs
mapred_all = MAPRED_DEFAULT
if CONF.enable_data_locality:
all_cfg.update(topology.TOPOLOGY_CONFIG)
# applying vm awareness configs
core_all += topology.vm_awareness_core_config()
mapred_all += topology.vm_awareness_mapred_config()
xml_configs = {
'core-site': x.create_hadoop_xml(all_cfg, core_all),
'mapred-site': x.create_hadoop_xml(all_cfg, mapred_all),
'hdfs-site': x.create_hadoop_xml(all_cfg, HDFS_DEFAULT)
}
if hive_hostname:
xml_configs.update({'hive-site':
x.create_hadoop_xml(all_cfg, HIVE_DEFAULT)})
LOG.debug('Generated hive-site.xml for hive % s', hive_hostname)
if oozie_hostname:
xml_configs.update({'oozie-site':
x.create_hadoop_xml(all_cfg, o_h.OOZIE_DEFAULT)})
LOG.debug('Generated oozie-site.xml for oozie % s', oozie_hostname)
return xml_configs
def get_general_configs(hive_hostname, passwd_hive_mysql):
config = {
ENABLE_SWIFT.name: {
'default_value': ENABLE_SWIFT.default_value,
'conf': extract_name_values(swift.get_swift_configs())
},
ENABLE_MYSQL.name: {
'default_value': ENABLE_MYSQL.default_value,
'conf': m_h.get_required_mysql_configs(
hive_hostname, passwd_hive_mysql)
}
}
if CONF.enable_data_locality:
config.update({
ENABLE_DATA_LOCALITY.name: {
'default_value': ENABLE_DATA_LOCALITY.default_value,
'conf': extract_name_values(topology.vm_awareness_all_config())
}
})
return config
def get_general_configs(hive_hostname, passwd_hive_mysql):
config = {
ENABLE_SWIFT.name: {
'default_value': ENABLE_SWIFT.default_value,
'conf': extract_name_values(swift.get_swift_configs())
},
ENABLE_MYSQL.name: {
'default_value':
ENABLE_MYSQL.default_value,
'conf':
m_h.get_required_mysql_configs(hive_hostname, passwd_hive_mysql)
}
}
if CONF.enable_data_locality:
config.update({
ENABLE_DATA_LOCALITY.name: {
'default_value': ENABLE_DATA_LOCALITY.default_value,
'conf': extract_name_values(topology.vm_awareness_all_config())
}
})
return config
def generate_xml_configs(configs, storage_path, nn_hostname,
jt_hostname, oozie_hostname, hive_hostname):
set_general_configs(hive_hostname)
# inserting common configs depends on provisioned VMs and HDFS placement
# TODO(aignatov): should be moved to cluster context
cfg = {
'fs.default.name': 'hdfs://%s:8020' % nn_hostname,
'dfs.name.dir': extract_hadoop_path(storage_path,
'/lib/hadoop/hdfs/namenode'),
'dfs.data.dir': extract_hadoop_path(storage_path,
'/lib/hadoop/hdfs/datanode'),
'dfs.hosts': '/etc/hadoop/dn.incl',
'dfs.hosts.exclude': '/etc/hadoop/dn.excl',
}
if jt_hostname:
mr_cfg = {
'mapred.job.tracker': '%s:8021' % jt_hostname,
'mapred.system.dir': extract_hadoop_path(storage_path,
'/mapred/mapredsystem'),
'mapred.local.dir': extract_hadoop_path(storage_path,
'/lib/hadoop/mapred'),
'mapred.hosts': '/etc/hadoop/tt.incl',
'mapred.hosts.exclude': '/etc/hadoop/tt.excl',
}
cfg.update(mr_cfg)
if oozie_hostname:
o_cfg = {
'hadoop.proxyuser.hadoop.hosts': "localhost," + oozie_hostname,
'hadoop.proxyuser.hadoop.groups': 'hadoop',
}
cfg.update(o_cfg)
LOG.debug('Applied Oozie configs for core-site.xml')
cfg.update(o_h.get_oozie_required_xml_configs())
LOG.debug('Applied Oozie configs for oozie-site.xml')
if hive_hostname:
h_cfg = {
'hive.warehouse.subdir.inherit.perms': True,
'javax.jdo.option.ConnectionURL':
'jdbc:derby:;databaseName=/opt/hive/metastore_db;create=true'
}
cfg.update(h_cfg)
LOG.debug('Applied Hive config for hive metastore server')
# inserting user-defined configs
for key, value in extract_xml_confs(configs):
cfg[key] = value
# applying swift configs if user enabled it
swift_xml_confs = swift.get_swift_configs()
cfg = generate_cfg_from_general(cfg, configs, GENERAL_CONFS)
# invoking applied configs to appropriate xml files
xml_configs = {
'core-site': x.create_hadoop_xml(cfg, CORE_DEFAULT + swift_xml_confs),
'mapred-site': x.create_hadoop_xml(cfg, MAPRED_DEFAULT),
'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT)
}
if hive_hostname:
xml_configs.update({'hive-site':
x.create_hadoop_xml(cfg, HIVE_DEFAULT)})
LOG.debug('Generated hive-site.xml for hive % s', hive_hostname)
if oozie_hostname:
xml_configs.update({'oozie-site':
x.create_hadoop_xml(cfg, o_h.OOZIE_DEFAULT)})
LOG.debug('Generated oozie-site.xml for oozie % s', oozie_hostname)
return xml_configs
def _get_swift_properties(self):
return h.get_swift_configs()
def generate_xml_configs(configs, storage_path, nn_hostname, jt_hostname,
oozie_hostname):
# inserting common configs depends on provisioned VMs and HDFS placement
# TODO(aignatov): should be moved to cluster context
cfg = {
'fs.default.name':
'hdfs://%s:8020' % nn_hostname,
'dfs.name.dir':
extract_hadoop_path(storage_path, '/lib/hadoop/hdfs/namenode'),
'dfs.data.dir':
extract_hadoop_path(storage_path, '/lib/hadoop/hdfs/datanode'),
'dfs.hosts':
'/etc/hadoop/dn.incl',
'dfs.hosts.exclude':
'/etc/hadoop/dn.excl',
}
if jt_hostname:
mr_cfg = {
'mapred.job.tracker':
'%s:8021' % jt_hostname,
'mapred.system.dir':
extract_hadoop_path(storage_path, '/mapred/mapredsystem'),
'mapred.local.dir':
extract_hadoop_path(storage_path, '/lib/hadoop/mapred'),
'mapred.hosts':
'/etc/hadoop/tt.incl',
'mapred.hosts.exclude':
'/etc/hadoop/tt.excl',
}
cfg.update(mr_cfg)
if oozie_hostname:
o_cfg = {
'hadoop.proxyuser.hadoop.hosts': "localhost," + oozie_hostname,
'hadoop.proxyuser.hadoop.groups': 'hadoop',
}
cfg.update(o_cfg)
LOG.debug('Applied Oozie configs for core-site.xml')
cfg.update(o_h.get_oozie_required_xml_configs())
LOG.debug('Applied Oozie configs for oozie-site.xml')
# inserting user-defined configs
for key, value in extract_xml_confs(configs):
cfg[key] = value
# applying swift configs if user enabled it
swift_xml_confs = []
#TODO(aignatov): should be changed. General configs not only Swift
swift_in_config = False
if ('general' in configs and ENABLE_SWIFT.name in configs['general']):
swift_in_config = True
if ((swift_in_config and configs['general'][ENABLE_SWIFT.name])
or (not swift_in_config and ENABLE_SWIFT.default_value)):
swift_xml_confs = swift.get_swift_configs()
cfg.update(extract_name_values(swift_xml_confs))
LOG.info("Swift integration is enabled")
# invoking applied configs to appropriate xml files
xml_configs = {
'core-site': x.create_hadoop_xml(cfg, CORE_DEFAULT + swift_xml_confs),
'mapred-site': x.create_hadoop_xml(cfg, MAPRED_DEFAULT),
'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT)
}
if oozie_hostname:
xml_configs.update(
{'oozie-site': x.create_hadoop_xml(cfg, o_h.OOZIE_DEFAULT)})
LOG.debug('Generated oozie-site.xml for oozie % s', oozie_hostname)
return xml_configs
def generate_xml_configs(configs, storage_path, nn_hostname, jt_hostname,
oozie_hostname, hive_hostname, passwd_hive_mysql):
general_cfg = get_general_configs(hive_hostname, passwd_hive_mysql)
# inserting common configs depends on provisioned VMs and HDFS placement
# TODO(aignatov): should be moved to cluster context
cfg = {
'fs.default.name':
'hdfs://%s:8020' % nn_hostname,
'dfs.name.dir':
extract_hadoop_path(storage_path, '/lib/hadoop/hdfs/namenode'),
'dfs.data.dir':
extract_hadoop_path(storage_path, '/lib/hadoop/hdfs/datanode'),
'dfs.hosts':
'/etc/hadoop/dn.incl',
'dfs.hosts.exclude':
'/etc/hadoop/dn.excl',
}
if jt_hostname:
mr_cfg = {
'mapred.job.tracker':
'%s:8021' % jt_hostname,
'mapred.system.dir':
extract_hadoop_path(storage_path, '/mapred/mapredsystem'),
'mapred.local.dir':
extract_hadoop_path(storage_path, '/lib/hadoop/mapred'),
'mapred.hosts':
'/etc/hadoop/tt.incl',
'mapred.hosts.exclude':
'/etc/hadoop/tt.excl',
}
cfg.update(mr_cfg)
if oozie_hostname:
o_cfg = {
'hadoop.proxyuser.hadoop.hosts': "localhost," + oozie_hostname,
'hadoop.proxyuser.hadoop.groups': 'hadoop',
}
cfg.update(o_cfg)
LOG.debug('Applied Oozie configs for core-site.xml')
cfg.update(o_h.get_oozie_required_xml_configs())
LOG.debug('Applied Oozie configs for oozie-site.xml')
if hive_hostname:
h_cfg = {
'hive.warehouse.subdir.inherit.perms':
True,
'javax.jdo.option.ConnectionURL':
'jdbc:derby:;databaseName=/opt/hive/metastore_db;create=true'
}
cfg.update(h_cfg)
LOG.debug('Applied Hive config for hive metastore server')
# inserting user-defined configs
for key, value in extract_xml_confs(configs):
cfg[key] = value
# applying swift configs if user enabled it
swift_xml_confs = swift.get_swift_configs()
cfg = generate_cfg_from_general(cfg, configs, general_cfg)
# invoking applied configs to appropriate xml files
core_all = CORE_DEFAULT + swift_xml_confs
mapred_all = MAPRED_DEFAULT
if CONF.enable_data_locality:
cfg.update(topology.TOPOLOGY_CONFIG)
# applying vm awareness configs
core_all += topology.vm_awareness_core_config()
mapred_all += topology.vm_awareness_mapred_config()
xml_configs = {
'core-site': x.create_hadoop_xml(cfg, core_all),
'mapred-site': x.create_hadoop_xml(cfg, mapred_all),
'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT)
}
if hive_hostname:
xml_configs.update(
{'hive-site': x.create_hadoop_xml(cfg, HIVE_DEFAULT)})
LOG.debug('Generated hive-site.xml for hive % s', hive_hostname)
if oozie_hostname:
xml_configs.update(
{'oozie-site': x.create_hadoop_xml(cfg, o_h.OOZIE_DEFAULT)})
LOG.debug('Generated oozie-site.xml for oozie % s', oozie_hostname)
return xml_configs
def _get_swift_properties(self):
return h.get_swift_configs()
