def get_possible_job_config(job_type):
if not edp.compare_job_type(job_type, *get_possible_job_types()):
return None
if edp.compare_job_type(job_type, 'Java'):
return {'job_config': {'configs': [], 'args': []}}
if edp.compare_job_type(job_type, 'MapReduce', 'Pig'):
#TODO(nmakhotkin) Savanna should return config based on specific plugin
cfg = xmlutils.load_hadoop_xml_defaults(
'plugins/vanilla/v1_2_1/resources/mapred-default.xml')
if edp.compare_job_type(job_type, 'MapReduce'):
cfg += xmlutils.load_hadoop_xml_defaults(
'service/edp/resources/mapred-job-config.xml')
elif edp.compare_job_type(job_type, 'Hive'):
#TODO(nmakhotkin) Savanna should return config based on specific plugin
cfg = xmlutils.load_hadoop_xml_defaults(
'plugins/vanilla/v1_2_1/resources/hive-default.xml')
# TODO(tmckay): args should be a list when bug #269968
# is fixed on the UI side
config = {'configs': cfg, "args": {}}
if not edp.compare_job_type('MapReduce', 'Java'):
config.update({'params': {}})
return {'job_config': config}
def get_possible_job_config(job_type):
if job_type not in get_possible_job_types():
return None
if job_type == "Java":
return {'job_config': {'configs': [], 'args': []}}
if job_type in ['MapReduce', 'Pig', 'Jar']:
#TODO(nmakhotkin) Savanna should return config based on specific plugin
cfg = xmlutils.load_hadoop_xml_defaults(
'plugins/vanilla/resources/mapred-default.xml')
if job_type in ['MapReduce', 'Jar']:
cfg += xmlutils.load_hadoop_xml_defaults(
'service/edp/resources/mapred-job-config.xml')
elif job_type == 'Hive':
#TODO(nmakhotkin) Savanna should return config based on specific plugin
cfg = xmlutils.load_hadoop_xml_defaults(
'plugins/vanilla/resources/hive-default.xml')
# TODO(tmckay): args should be a list when bug #269968
# is fixed on the UI side
config = {'configs': cfg, "args": {}}
if job_type not in ['MapReduce', 'Jar', 'Java']:
config.update({'params': {}})
return {'job_config': config}
def vm_awareness_mapred_config():
c = x.load_hadoop_xml_defaults('topology/resources/mapred-template.xml')
result = [cfg for cfg in c if cfg['value']]
LOG.info(
"Vm awareness will add following configs in map-red "
"params: %s", result)
return result
def test_load_xml_defaults(self):
self.assertListEqual(
[{'name': u'name1', 'value': u'value1', 'description': 'descr1'},
{'name': u'name2', 'value': u'value2', 'description': 'descr2'},
{'name': u'name3', 'value': '', 'description': 'descr3'},
{'name': u'name4', 'value': '', 'description': 'descr4'},
{'name': u'name5', 'value': u'value5', 'description': ''}],
x.load_hadoop_xml_defaults(
'tests/unit/resources/test-default.xml'))
def get_possible_job_config(job_type):
if job_type not in get_possible_job_types():
return None
if job_type in ['Jar', 'Pig']:
#TODO(nmakhotkin) Savanna should return config based on specific plugin
cfg = xmlutils.load_hadoop_xml_defaults(
'plugins/vanilla/resources/mapred-default.xml')
if job_type == 'Jar':
cfg += xmlutils.load_hadoop_xml_defaults(
'service/edp/resources/mapred-job-config.xml')
elif job_type == 'Hive':
#TODO(nmakhotkin) Savanna should return config based on specific plugin
cfg = xmlutils.load_hadoop_xml_defaults(
'plugins/vanilla/resources/hive-default.xml')
config = {'configs': cfg, "args": {}}
if job_type != 'Jar':
config.update({'params': {}})
return {'job_config': config}
def get_possible_job_config(job_type):
if job_type not in get_possible_job_types():
return None
if job_type in ['Jar', 'Pig']:
#TODO(nmakhotkin) Savanna should return config based on specific plugin
cfg = xmlutils.load_hadoop_xml_defaults(
'plugins/vanilla/resources/mapred-default.xml')
if job_type == 'Jar':
cfg += xmlutils.load_hadoop_xml_defaults(
'service/edp/resources/mapred-job-config.xml')
elif job_type == 'Hive':
#TODO(nmakhotkin) Savanna should return config based on specific plugin
cfg = xmlutils.load_hadoop_xml_defaults(
'plugins/vanilla/resources/hive-default.xml')
config = {'configs': cfg, "args": {}}
if job_type != 'Jar':
config.update({'params': {}})
return {'job_config': config}
def get_swift_configs():
configs = x.load_hadoop_xml_defaults("swift/resources/conf-template.xml")
for conf in configs:
if conf["name"] == HADOOP_SWIFT_AUTH_URL:
conf["value"] = _retrieve_auth_url()
if conf["name"] == HADOOP_SWIFT_TENANT:
conf["value"] = _retrieve_tenant()
result = [cfg for cfg in configs if cfg["value"]]
LOG.info("Swift would be integrated with the following " "params: %s", result)
return result
def get_swift_configs():
configs = x.load_hadoop_xml_defaults('swift/resources/conf-template.xml')
for conf in configs:
if conf['name'] == HADOOP_SWIFT_AUTH_URL:
conf['value'] = su.retrieve_auth_url() + "tokens/"
if conf['name'] == HADOOP_SWIFT_TENANT:
conf['value'] = _retrieve_tenant()
result = [cfg for cfg in configs if cfg['value']]
LOG.info("Swift would be integrated with the following "
"params: %s", result)
return result
def vm_awareness_core_config():
c = x.load_hadoop_xml_defaults('topology/resources/core-template.xml')
result = [cfg for cfg in c if cfg['value']]
if not CONF.enable_hypervisor_awareness:
# not leveraging 4-layer approach so override template value
param = next((prop for prop in result
if prop['name'] == 'net.topology.impl'), None)
if param:
param['value'] = 'org.apache.hadoop.net.NetworkTopology'
LOG.info("Vm awareness will add following configs in core-site "
"params: %s", result)
return result
def vm_awareness_core_config():
c = x.load_hadoop_xml_defaults('topology/resources/core-template.xml')
result = [cfg for cfg in c if cfg['value']]
if not CONF.enable_hypervisor_awareness:
# not leveraging 4-layer approach so override template value
param = next(
(prop for prop in result if prop['name'] == 'net.topology.impl'),
None)
if param:
param['value'] = 'org.apache.hadoop.net.NetworkTopology'
LOG.info(
"Vm awareness will add following configs in core-site "
"params: %s", result)
return result
def test_create_hadoop_xml(self):
conf = x.load_hadoop_xml_defaults(
'tests/unit/resources/test-default.xml')
self.assertEquals(x.create_hadoop_xml({'name1': 'some_val1',
'name2': 2}, conf),
"""<?xml version="1.0" ?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>name2</name>
<value>2</value>
</property>
<property>
<name>name1</name>
<value>some_val1</value>
</property>
</configuration>
""")
def test_create_hadoop_xml(self):
conf = x.load_hadoop_xml_defaults(
'tests/unit/resources/test-default.xml')
self.assertEqual(
x.create_hadoop_xml({
'name1': 'some_val1',
'name2': 2
}, conf), """<?xml version="1.0" ?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>name2</name>
<value>2</value>
</property>
<property>
<name>name1</name>
<value>some_val1</value>
</property>
</configuration>
""")
def test_load_xml_defaults(self):
self.assertListEqual([
{
'name': u'name1',
'value': u'value1',
'description': 'descr1'
}, {
'name': u'name2',
'value': u'value2',
'description': 'descr2'
}, {
'name': u'name3',
'value': '',
'description': 'descr3'
}, {
'name': u'name4',
'value': '',
'description': 'descr4'
}, {
'name': u'name5',
'value': u'value5',
'description': ''
}
], x.load_hadoop_xml_defaults('tests/unit/resources/test-default.xml'))
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from savanna.openstack.common import log as logging
from savanna.plugins import provisioning as p
from savanna.plugins.vanilla import oozie_helper as o_h
from savanna.swift import swift_helper as swift
from savanna.utils import xmlutils as x
LOG = logging.getLogger(__name__)
CORE_DEFAULT = x.load_hadoop_xml_defaults(
'plugins/vanilla/resources/core-default.xml')
HDFS_DEFAULT = x.load_hadoop_xml_defaults(
'plugins/vanilla/resources/hdfs-default.xml')
MAPRED_DEFAULT = x.load_hadoop_xml_defaults(
'plugins/vanilla/resources/mapred-default.xml')
## Append Oozie configs fore core-site.xml
CORE_DEFAULT += o_h.OOZIE_CORE_DEFAULT
XML_CONFS = {
"HDFS": [CORE_DEFAULT, HDFS_DEFAULT],
"MapReduce": [MAPRED_DEFAULT],
"JobFlow": [o_h.OOZIE_DEFAULT]
}
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from savanna.utils import xmlutils as x
OOZIE_DEFAULT = x.load_hadoop_xml_defaults(
'plugins/vanilla/resources/oozie-default.xml')
OOZIE_CORE_DEFAULT = [{
'name': 'hadoop.proxyuser.hadoop.hosts',
'value': "localhost"
}, {
'name': 'hadoop.proxyuser.hadoop.groups',
'value': 'hadoop'
}]
def get_oozie_required_xml_configs():
"""Following configs differ from default configs in oozie-default.xml."""
return {
'oozie.service.ActionService.executor.ext.classes':
'org.apache.oozie.action.email.EmailActionExecutor,'
# limitations under the License.
from savanna.plugins import provisioning as p
from savanna.utils import xmlutils as x
CORE_DEFAULT = x.load_hadoop_xml_defaults_with_type_and_locale(
'plugins/intel/resources/hadoop-default.xml')
HDFS_DEFAULT = x.load_hadoop_xml_defaults_with_type_and_locale(
'plugins/intel/resources/hdfs-default.xml')
MAPRED_DEFAULT = x.load_hadoop_xml_defaults_with_type_and_locale(
'plugins/intel/resources/mapred-default.xml')
OOZIE_DEFAULT = x.load_hadoop_xml_defaults(
'plugins/intel/resources/oozie-default.xml')
XML_CONFS = {
"Hadoop": [CORE_DEFAULT],
"HDFS": [HDFS_DEFAULT],
"MapReduce": [MAPRED_DEFAULT],
"JobFlow": [OOZIE_DEFAULT]
}
IDH_TARBALL_URL = p.Config('IDH tarball URL', 'general', 'cluster', priority=1,
default_value='http://repo1.intelhadoop.com:3424/'
'setup/setup-intelhadoop-'
'2.5.1-en-evaluation.RHEL.tar.gz')
OS_REPO_URL = p.Config('OS repository URL', 'general', 'cluster', priority=1,
def vm_awareness_mapred_config():
c = x.load_hadoop_xml_defaults('topology/resources/mapred-template.xml')
result = [cfg for cfg in c if cfg['value']]
LOG.info("Vm awareness will add following configs in map-red "
"params: %s", result)
return result
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from savanna.openstack.common import log as logging
from savanna.plugins import provisioning as p
from savanna.plugins.vanilla import mysql_helper as m_h
from savanna.plugins.vanilla import oozie_helper as o_h
from savanna.swift import swift_helper as swift
from savanna.utils import xmlutils as x
LOG = logging.getLogger(__name__)
CORE_DEFAULT = x.load_hadoop_xml_defaults(
'plugins/vanilla/resources/core-default.xml')
HDFS_DEFAULT = x.load_hadoop_xml_defaults(
'plugins/vanilla/resources/hdfs-default.xml')
MAPRED_DEFAULT = x.load_hadoop_xml_defaults(
'plugins/vanilla/resources/mapred-default.xml')
HIVE_DEFAULT = x.load_hadoop_xml_defaults(
'plugins/vanilla/resources/hive-default.xml')
## Append Oozie configs fore core-site.xml
CORE_DEFAULT += o_h.OOZIE_CORE_DEFAULT
XML_CONFS = {
"HDFS": [CORE_DEFAULT, HDFS_DEFAULT],
# See the License for the specific language governing permissions and
# limitations under the License.
from savanna.plugins import provisioning as p
from savanna.utils import xmlutils as x
CORE_DEFAULT = x.load_hadoop_xml_defaults_with_type_and_locale(
'plugins/intel/v2_5_1/resources/hadoop-default.xml')
HDFS_DEFAULT = x.load_hadoop_xml_defaults_with_type_and_locale(
'plugins/intel/v2_5_1/resources/hdfs-default.xml')
MAPRED_DEFAULT = x.load_hadoop_xml_defaults_with_type_and_locale(
'plugins/intel/v2_5_1/resources/mapred-default.xml')
OOZIE_DEFAULT = x.load_hadoop_xml_defaults(
'plugins/intel/v2_5_1/resources/oozie-default.xml')
XML_CONFS = {
"Hadoop": [CORE_DEFAULT],
"HDFS": [HDFS_DEFAULT],
"MapReduce": [MAPRED_DEFAULT],
"JobFlow": [OOZIE_DEFAULT]
}
IDH_TARBALL_URL = p.Config('IDH tarball URL',
'general',
'cluster',
priority=1,
default_value='http://repo1.intelhadoop.com:3424/'
'setup/setup-intelhadoop-'
'2.5.1-en-evaluation.RHEL.tar.gz')
