def _extract_configs_to_extra(self, cluster):
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
oozie = utils.get_oozie(cluster)
hive = utils.get_hiveserver(cluster)
extra = dict()
if hive:
extra['hive_mysql_passwd'] = uuidutils.generate_uuid()
for ng in cluster.node_groups:
extra[ng.id] = {
'xml':
c_helper.generate_xml_configs(
ng.configuration, ng.storage_paths, nn.hostname,
jt.hostname if jt else None,
oozie.hostname if oozie else None,
hive.hostname if hive else None,
extra['hive_mysql_passwd'] if hive else None),
'setup_script':
c_helper.generate_setup_script(
ng.storage_paths,
c_helper.extract_environment_confs(ng.configuration),
append_oozie=(oozie is not None
and oozie.node_group.id == ng.id))
}
if c_helper.is_data_locality_enabled(cluster):
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
extra['topology_data'] = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
return extra
def _extract_configs_to_extra(self, cluster):
oozie = utils.get_oozie(cluster)
hive = utils.get_hiveserver(cluster)
extra = dict()
if hive:
extra['hive_mysql_passwd'] = six.text_type(uuid.uuid4())
for ng in cluster.node_groups:
extra[ng.id] = {
'xml': c_helper.generate_xml_configs(
cluster, ng, extra['hive_mysql_passwd'] if hive else None),
'setup_script': c_helper.generate_setup_script(
ng.storage_paths(),
c_helper.extract_environment_confs(ng.configuration()),
append_oozie=(
oozie and oozie.node_group.id == ng.id)
)
}
if c_helper.is_data_locality_enabled(cluster):
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
extra['topology_data'] = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
return extra
def _extract_configs_to_extra(self, cluster):
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
oozie = utils.get_oozie(cluster)
hive = utils.get_hiveserver(cluster)
extra = dict()
for ng in cluster.node_groups:
extra[ng.id] = {
'xml': c_helper.generate_xml_configs(ng.configuration,
ng.storage_paths,
nn.hostname,
jt.hostname
if jt else None,
oozie.hostname
if oozie else None,
hive.hostname
if hive else None),
'setup_script': c_helper.generate_setup_script(
ng.storage_paths,
c_helper.extract_environment_confs(ng.configuration),
append_oozie=(
oozie is not None and oozie.node_group.id == ng.id)
)
}
return extra
def test_extract_environment_configs(self):
env_configs = {
"MapReduce": {"Job Tracker Heap Size": 1000, "Task Tracker Heap Size": "2000"},
"HDFS": {"Name Node Heap Size": 3000, "Data Node Heap Size": "4000"},
"Wrong-applicable-target": {"t1": 4},
}
self.assertListEqual(
c_h.extract_environment_confs(env_configs),
[
'HADOOP_NAMENODE_OPTS=\\"-Xmx3000m\\"',
'HADOOP_DATANODE_OPTS=\\"-Xmx4000m\\"',
'HADOOP_JOBTRACKER_OPTS=\\"-Xmx1000m\\"',
'HADOOP_TASKTRACKER_OPTS=\\"-Xmx2000m\\"',
],
)
def _extract_configs(self, cluster):
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
for ng in cluster.node_groups:
ng.extra = {
'xml': c_helper.generate_xml_configs(ng.configuration,
ng.storage_paths,
nn.hostname,
jt.hostname
if jt else None),
'setup_script': c_helper.generate_setup_script(
ng.storage_paths,
c_helper.extract_environment_confs(ng.configuration)
)
}
def _extract_configs_to_extra(self, cluster):
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
oozie = utils.get_oozie(cluster)
extra = dict()
for ng in cluster.node_groups:
extra[ng.id] = {
'xml':
c_helper.generate_xml_configs(
ng.configuration, ng.storage_paths, nn.hostname,
jt.hostname if jt else None,
oozie.hostname if oozie else None),
'setup_script':
c_helper.generate_setup_script(
ng.storage_paths,
c_helper.extract_environment_confs(ng.configuration))
}
return extra
def test_extract_environment_configs(self):
env_configs = {
"MapReduce": {
'Job Tracker Heap Size': 1000,
'Task Tracker Heap Size': "2000"
},
"HDFS": {
'Name Node Heap Size': 3000,
'Data Node Heap Size': "4000"
},
"Wrong-applicable-target": {
't1': 4
}
}
self.assertListEqual(c_h.extract_environment_confs(env_configs), [
'HADOOP_NAMENODE_OPTS=\\"-Xmx3000m\\"',
'HADOOP_DATANODE_OPTS=\\"-Xmx4000m\\"',
'HADOOP_JOBTRACKER_OPTS=\\"-Xmx1000m\\"',
'HADOOP_TASKTRACKER_OPTS=\\"-Xmx2000m\\"'
])
def _extract_configs_to_extra(self, cluster):
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
oozies_hostnames = [o.hostname for o in utils.get_oozies(cluster)]
extra = dict()
for ng in cluster.node_groups:
extra[ng.id] = {
'xml': c_helper.generate_xml_configs(ng.configuration,
ng.storage_paths,
nn.hostname,
jt.hostname
if jt else None,
oozies_hostnames),
'setup_script': c_helper.generate_setup_script(
ng.storage_paths,
c_helper.extract_environment_confs(ng.configuration)
)
}
return extra
def test_extract_environment_configs(self):
env_configs = {
"JobFlow": {
'Oozie Heap Size': 4000
},
"MapReduce": {
'Job Tracker Heap Size': 1000,
'Task Tracker Heap Size': "2000"
},
"HDFS": {
'Name Node Heap Size': 3000,
'Data Node Heap Size': "4000"
},
"Wrong-applicable-target": {
't1': 4
}}
self.assertListEqual(c_h.extract_environment_confs(env_configs),
['HADOOP_NAMENODE_OPTS=\\"-Xmx3000m\\"',
'HADOOP_DATANODE_OPTS=\\"-Xmx4000m\\"',
'CATALINA_OPTS=\\"-Xmx4000m\\"',
'HADOOP_JOBTRACKER_OPTS=\\"-Xmx1000m\\"',
'HADOOP_TASKTRACKER_OPTS=\\"-Xmx2000m\\"'])
def _extract_configs_to_extra(self, cluster):
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
oozie = utils.get_oozie(cluster)
hive = utils.get_hiveserver(cluster)
extra = dict()
if hive:
extra['hive_mysql_passwd'] = uuidutils.generate_uuid()
for ng in cluster.node_groups:
extra[ng.id] = {
'xml': c_helper.generate_xml_configs(
ng.configuration,
ng.storage_paths,
nn.hostname,
jt.hostname if jt else None,
oozie.hostname if oozie else None,
hive.hostname if hive else None,
extra['hive_mysql_passwd'] if hive else None),
'setup_script': c_helper.generate_setup_script(
ng.storage_paths,
c_helper.extract_environment_confs(ng.configuration),
append_oozie=(
oozie is not None and oozie.node_group.id == ng.id)
)
}
if c_helper.is_data_locality_enabled(cluster):
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
extra['topology_data'] = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
return extra
