def get_possible_job_config(job_type): if not edp.compare_job_type(job_type, *edp.JOB_TYPES_ALL): return None if edp.compare_job_type(job_type, edp.JOB_TYPE_JAVA): return {'job_config': {'configs': [], 'args': []}} if edp.compare_job_type(job_type, edp.JOB_TYPE_MAPREDUCE, edp.JOB_TYPE_PIG): # TODO(nmakhotkin): Here we need return config based on specific plugin cfg = xmlutils.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/mapred-default.xml') if edp.compare_job_type(job_type, edp.JOB_TYPE_MAPREDUCE): cfg += xmlutils.load_hadoop_xml_defaults( 'service/edp/resources/mapred-job-config.xml') elif edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE): # TODO(nmakhotkin): Here we need return config based on specific plugin cfg = xmlutils.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/hive-default.xml') # TODO(tmckay): args should be a list when bug #269968 # is fixed on the UI side config = {'configs': cfg, "args": {}} if not edp.compare_job_type(edp.JOB_TYPE_MAPREDUCE, edp.JOB_TYPE_JAVA): config.update({'params': {}}) return {'job_config': config}
def get_possible_job_config(job_type): if not edp.compare_job_type(job_type, *edp.JOB_TYPES_ALL): return None if edp.compare_job_type(job_type, edp.JOB_TYPE_JAVA): return {'job_config': {'configs': [], 'args': []}} if edp.compare_job_type(job_type, edp.JOB_TYPE_SHELL): return {'job_config': {'configs': [], 'params': {}, 'args': []}} if edp.compare_job_type(job_type, edp.JOB_TYPE_MAPREDUCE, edp.JOB_TYPE_PIG): cfg = xmlutils.load_hadoop_xml_defaults( 'service/edp/resources/mapred-default.xml') if edp.compare_job_type(job_type, edp.JOB_TYPE_MAPREDUCE): cfg += get_possible_mapreduce_configs() elif edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE): cfg = xmlutils.load_hadoop_xml_defaults( 'service/edp/resources/hive-default.xml') config = {'configs': cfg} if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG, edp.JOB_TYPE_HIVE): config.update({'params': {}}) if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG): config.update({'args': []}) return {'job_config': config}
def get_possible_job_config(job_type): if not edp.compare_job_type(job_type, *get_possible_job_types()): return None if edp.compare_job_type(job_type, 'Java'): return {'job_config': {'configs': [], 'args': []}} if edp.compare_job_type(job_type, 'MapReduce', 'Pig'): #TODO(nmakhotkin) Here we should return config based on specific plugin cfg = xmlutils.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/mapred-default.xml') if edp.compare_job_type(job_type, 'MapReduce'): cfg += xmlutils.load_hadoop_xml_defaults( 'service/edp/resources/mapred-job-config.xml') elif edp.compare_job_type(job_type, 'Hive'): #TODO(nmakhotkin) Here we should return config based on specific plugin cfg = xmlutils.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/hive-default.xml') # TODO(tmckay): args should be a list when bug #269968 # is fixed on the UI side config = {'configs': cfg, "args": {}} if not edp.compare_job_type('MapReduce', 'Java'): config.update({'params': {}}) return {'job_config': config}
def get_possible_job_config(job_type): if not edp.compare_job_type(job_type, *edp.JOB_TYPES_ALL): return None if edp.compare_job_type(job_type, edp.JOB_TYPE_JAVA): return {'job_config': {'configs': [], 'args': []}} if edp.compare_job_type(job_type, edp.JOB_TYPE_SHELL): return {'job_config': {'configs': [], 'params': [], 'args': []}} if edp.compare_job_type(job_type, edp.JOB_TYPE_MAPREDUCE, edp.JOB_TYPE_PIG): # TODO(nmakhotkin): Here we need return config based on specific plugin cfg = xmlutils.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/mapred-default.xml') if edp.compare_job_type(job_type, edp.JOB_TYPE_MAPREDUCE): cfg += xmlutils.load_hadoop_xml_defaults( 'service/edp/resources/mapred-job-config.xml') elif edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE): # TODO(nmakhotkin): Here we need return config based on specific plugin cfg = xmlutils.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/hive-default.xml') # TODO(tmckay): args should be a list when bug #269968 # is fixed on the UI side config = {'configs': cfg, "args": {}} if not edp.compare_job_type(edp.JOB_TYPE_MAPREDUCE, edp.JOB_TYPE_JAVA): config.update({'params': {}}) return {'job_config': config}
def get_possible_job_config(job_type): if not edp.compare_job_type(job_type, *edp.JOB_TYPES_ALL): return None if edp.compare_job_type(job_type, edp.JOB_TYPE_JAVA): return {'job_config': {'configs': [], 'args': []}} if edp.compare_job_type(job_type, edp.JOB_TYPE_SHELL): return {'job_config': {'configs': [], 'params': {}, 'args': []}} if edp.compare_job_type(job_type, edp.JOB_TYPE_MAPREDUCE, edp.JOB_TYPE_PIG): cfg = xmlutils.load_hadoop_xml_defaults( 'service/edp/resources/mapred-default.xml') if edp.compare_job_type(job_type, edp.JOB_TYPE_MAPREDUCE): cfg += get_possible_mapreduce_configs() elif edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE): cfg = xmlutils.load_hadoop_xml_defaults( 'service/edp/resources/hive-default.xml') config = {'configs': cfg} if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG, edp.JOB_TYPE_HIVE): config.update({'params': {}}) if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG): config.update({'args': []}) return {'job_config': config}
def get_possible_job_config(job_type): if not edp.compare_job_type(job_type, *edp.JOB_TYPES_ALL): return None if edp.compare_job_type(job_type, edp.JOB_TYPE_JAVA): return {'job_config': {'configs': [], 'args': []}} if edp.compare_job_type(job_type, edp.JOB_TYPE_SHELL): return {'job_config': {'configs': [], 'params': {}, 'args': []}} if edp.compare_job_type(job_type, edp.JOB_TYPE_MAPREDUCE, edp.JOB_TYPE_PIG): # TODO(nmakhotkin): Here we need return config based on specific plugin cfg = xmlutils.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/mapred-default.xml') if edp.compare_job_type(job_type, edp.JOB_TYPE_MAPREDUCE): cfg += get_possible_mapreduce_configs() elif edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE): # TODO(nmakhotkin): Here we need return config based on specific plugin cfg = xmlutils.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/hive-default.xml') config = {'configs': cfg} if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG, edp.JOB_TYPE_HIVE): config.update({'params': {}}) if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG): config.update({'args': []}) return {'job_config': config}
def get_possible_hive_config_from(file_name): '''Return the possible configs, args, params for a Hive job.''' config = { 'configs': xmlutils.load_hadoop_xml_defaults(file_name), 'params': {} } return config
def vm_awareness_mapred_config(): c = x.load_hadoop_xml_defaults('topology/resources/mapred-template.xml') result = [cfg for cfg in c if cfg['value']] LOG.info( "Vm awareness will add following configs in map-red " "params: %s", result) return result
def get_possible_hive_config_from(file_name): '''Return the possible configs, args, params for a Hive job.''' config = { 'configs': xmlutils.load_hadoop_xml_defaults(file_name), 'params': {} } return config
def test_load_xml_defaults(self): self.assertEqual( [{'name': u'name1', 'value': u'value1', 'description': 'descr1'}, {'name': u'name2', 'value': u'value2', 'description': 'descr2'}, {'name': u'name3', 'value': '', 'description': 'descr3'}, {'name': u'name4', 'value': '', 'description': 'descr4'}, {'name': u'name5', 'value': u'value5', 'description': ''}], x.load_hadoop_xml_defaults( 'tests/unit/resources/test-default.xml'))
def test_load_xml_defaults(self): self.assertEqual( [{'name': u'name1', 'value': u'value1', 'description': 'descr1'}, {'name': u'name2', 'value': u'value2', 'description': 'descr2'}, {'name': u'name3', 'value': '', 'description': 'descr3'}, {'name': u'name4', 'value': '', 'description': 'descr4'}, {'name': u'name5', 'value': u'value5', 'description': ''}], x.load_hadoop_xml_defaults( 'tests/unit/resources/test-default.xml'))
def get_swift_configs(): configs = x.load_hadoop_xml_defaults('swift/resources/conf-template.xml') for conf in configs: if conf['name'] == HADOOP_SWIFT_AUTH_URL: conf['value'] = su.retrieve_auth_url() + "tokens/" if conf['name'] == HADOOP_SWIFT_TENANT: conf['value'] = retrieve_tenant() result = [cfg for cfg in configs if cfg['value']] LOG.info("Swift would be integrated with the following " "params: %s", result) return result
def get_swift_configs(): configs = x.load_hadoop_xml_defaults("swift/resources/conf-template.xml") for conf in configs: if conf["name"] == HADOOP_SWIFT_AUTH_URL: conf["value"] = su.retrieve_auth_url() + "tokens/" if conf["name"] == HADOOP_SWIFT_TENANT: conf["value"] = retrieve_tenant() if CONF.os_region_name and conf["name"] == HADOOP_SWIFT_REGION: conf["value"] = CONF.os_region_name result = [cfg for cfg in configs if cfg["value"]] LOG.info(_LI("Swift would be integrated with the following " "params: {result}").format(result=result)) return result
def get_swift_configs(): configs = x.load_hadoop_xml_defaults('swift/resources/conf-template.xml') for conf in configs: if conf['name'] == HADOOP_SWIFT_AUTH_URL: conf['value'] = su.retrieve_auth_url() + "auth/tokens/" if conf['name'] == HADOOP_SWIFT_TENANT: conf['value'] = retrieve_tenant() if CONF.os_region_name and conf['name'] == HADOOP_SWIFT_REGION: conf['value'] = CONF.os_region_name result = [cfg for cfg in configs if cfg['value']] LOG.info("Swift would be integrated with the following " "params: {result}".format(result=result)) return result
def vm_awareness_core_config(): c = x.load_hadoop_xml_defaults('topology/resources/core-template.xml') result = [cfg for cfg in c if cfg['value']] if not CONF.enable_hypervisor_awareness: # not leveraging 4-layer approach so override template value param = next((prop for prop in result if prop['name'] == 'net.topology.impl'), None) if param: param['value'] = 'org.apache.hadoop.net.NetworkTopology' LOG.info(_LI("Vm awareness will add following configs in core-site " "params: %s"), result) return result
def vm_awareness_core_config(): c = x.load_hadoop_xml_defaults('topology/resources/core-template.xml') result = [cfg for cfg in c if cfg['value']] if not CONF.enable_hypervisor_awareness: # not leveraging 4-layer approach so override template value param = next((prop for prop in result if prop['name'] == 'net.topology.impl'), None) if param: param['value'] = 'org.apache.hadoop.net.NetworkTopology' LOG.info("Vm awareness will add following configs in core-site " "params: %s", result) return result
def test_create_hadoop_xml(self): conf = x.load_hadoop_xml_defaults( 'tests/unit/resources/test-default.xml') self.assertEqual("""<?xml version="1.0" ?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <configuration> <property> <name>name1</name> <value>some_val1</value> </property> <property> <name>name2</name> <value>2</value> </property> </configuration> """, x.create_hadoop_xml({'name1': 'some_val1', 'name2': 2}, conf),)
def test_create_hadoop_xml(self): conf = x.load_hadoop_xml_defaults( 'tests/unit/resources/test-default.xml') self.assertEqual(x.create_hadoop_xml({'name1': 'some_val1', 'name2': 2}, conf), """<?xml version="1.0" ?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <configuration> <property> <name>name2</name> <value>2</value> </property> <property> <name>name1</name> <value>some_val1</value> </property> </configuration> """)
def get_swift_configs(): configs = x.load_hadoop_xml_defaults('swift/resources/conf-template.xml') for conf in configs: if conf['name'] == HADOOP_SWIFT_AUTH_URL: conf['value'] = su.retrieve_auth_url() + "auth/tokens/" if conf['name'] == HADOOP_SWIFT_TENANT: conf['value'] = retrieve_tenant() if CONF.os_region_name and conf['name'] == HADOOP_SWIFT_REGION: conf['value'] = CONF.os_region_name if conf['name'] == HADOOP_SWIFT_DOMAIN_NAME: # NOTE(jfreud): Don't be deceived here... Even though there is an # attribute provided by context called domain_name, it is used for # domain scope, and hadoop-swiftfs always authenticates using # project scope. The purpose of the setting below is to override # the default value for project domain and user domain, domain id # as 'default', which may not always be correct. # TODO(jfreud): When hadoop-swiftfs allows it, stop hoping that # project_domain_name is always equal to user_domain_name. conf['value'] = context.current().project_domain_name result = [cfg for cfg in configs if cfg['value']] LOG.info("Swift would be integrated with the following " "params: {result}".format(result=result)) return result
def get_swift_configs(): configs = x.load_hadoop_xml_defaults('swift/resources/conf-template.xml') for conf in configs: if conf['name'] == HADOOP_SWIFT_AUTH_URL: conf['value'] = su.retrieve_auth_url() + "auth/tokens/" if conf['name'] == HADOOP_SWIFT_TENANT: conf['value'] = retrieve_tenant() if CONF.os_region_name and conf['name'] == HADOOP_SWIFT_REGION: conf['value'] = CONF.os_region_name if conf['name'] == HADOOP_SWIFT_DOMAIN_NAME: # NOTE(jfreud): Don't be deceived here... Even though there is an # attribute provided by context called domain_name, it is used for # domain scope, and hadoop-swiftfs always authenticates using # project scope. The purpose of the setting below is to override # the default value for project domain and user domain, domain id # as 'default', which may not always be correct. # TODO(jfreud): When hadoop-swiftfs allows it, stop hoping that # project_domain_name is always equal to user_domain_name. conf['value'] = context.current().project_domain_name result = [cfg for cfg in configs if cfg['value']] LOG.info("Swift would be integrated with the following " "params: {result}".format(result=result)) return result
# implied. # See the License for the specific language governing permissions and # limitations under the License. from oslo.config import cfg from sahara.openstack.common import log as logging from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper from sahara.utils import xmlutils as x CONF = cfg.CONF CONF.import_opt("enable_data_locality", "sahara.topology.topology_helper") LOG = logging.getLogger(__name__) CORE_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v2_4_1/resources/core-default.xml') HDFS_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v2_4_1/resources/hdfs-default.xml') MAPRED_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v2_4_1/resources/mapred-default.xml') YARN_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v2_4_1/resources/yarn-default.xml') OOZIE_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v2_4_1/resources/oozie-default.xml') XML_CONFS = { "Hadoop": [CORE_DEFAULT],
def read_default_swift_configs(): return x.load_hadoop_xml_defaults('swift/resources/conf-template.xml')
from oslo.config import cfg from sahara import conductor as c from sahara.openstack.common import log as logging from sahara.plugins.general import utils from sahara.plugins import provisioning as p from sahara.topology import topology_helper as topology from sahara.utils import types as types from sahara.utils import xmlutils as x conductor = c.API LOG = logging.getLogger(__name__) CONF = cfg.CONF CORE_DEFAULT = x.load_hadoop_xml_defaults("plugins/spark/resources/core-default.xml") HDFS_DEFAULT = x.load_hadoop_xml_defaults("plugins/spark/resources/hdfs-default.xml") XML_CONFS = {"HDFS": [CORE_DEFAULT, HDFS_DEFAULT]} SPARK_CONFS = { "Spark": { "OPTIONS": [ { "name": "Master port", "description": "Start the master on a different port" " (default: 7077)", "default": "7077", "priority": 2, }, {
# limitations under the License. from sahara.plugins import provisioning as p from sahara.utils import xmlutils as x CORE_DEFAULT = x.load_hadoop_xml_defaults_with_type_and_locale( 'plugins/intel/v2_5_1/resources/hadoop-default.xml') HDFS_DEFAULT = x.load_hadoop_xml_defaults_with_type_and_locale( 'plugins/intel/v2_5_1/resources/hdfs-default.xml') MAPRED_DEFAULT = x.load_hadoop_xml_defaults_with_type_and_locale( 'plugins/intel/v2_5_1/resources/mapred-default.xml') OOZIE_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/intel/v2_5_1/resources/oozie-default.xml') XML_CONFS = { "Hadoop": [CORE_DEFAULT], "HDFS": [HDFS_DEFAULT], "MapReduce": [MAPRED_DEFAULT], "JobFlow": [OOZIE_DEFAULT] } IDH_TARBALL_URL = p.Config('IDH tarball URL', 'general', 'cluster', priority=1, default_value='http://repo2.intelhadoop.com/' 'setup/setup-intelhadoop-' '2.5.1-en-evaluation.RHEL.tar.gz') OS_REPO_URL = p.Config('OS repository URL', 'general', 'cluster', priority=1,
# you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from sahara.utils import xmlutils as x OOZIE_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/oozie-default.xml') OOZIE_CORE_DEFAULT = [ { 'name': 'hadoop.proxyuser.hadoop.hosts', 'value': "localhost" }, { 'name': 'hadoop.proxyuser.hadoop.groups', 'value': 'hadoop' }] OOZIE_HEAPSIZE_DEFAULT = "CATALINA_OPTS -Xmx1024m" def get_oozie_required_xml_configs():
def vm_awareness_mapred_config(): c = x.load_hadoop_xml_defaults('topology/resources/mapred-template.xml') result = [cfg for cfg in c if cfg['value']] LOG.info("Vm awareness will add following configs in map-red " "params: %s", result) return result
# implied. # See the License for the specific language governing permissions and # limitations under the License. from oslo_config import cfg from oslo_log import log as logging from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper from sahara.utils import xmlutils as x CONF = cfg.CONF CONF.import_opt("enable_data_locality", "sahara.topology.topology_helper") LOG = logging.getLogger(__name__) CORE_DEFAULT = x.load_hadoop_xml_defaults("plugins/vanilla/v2_6_0/resources/core-default.xml") HDFS_DEFAULT = x.load_hadoop_xml_defaults("plugins/vanilla/v2_6_0/resources/hdfs-default.xml") MAPRED_DEFAULT = x.load_hadoop_xml_defaults("plugins/vanilla/v2_6_0/resources/mapred-default.xml") YARN_DEFAULT = x.load_hadoop_xml_defaults("plugins/vanilla/v2_6_0/resources/yarn-default.xml") OOZIE_DEFAULT = x.load_hadoop_xml_defaults("plugins/vanilla/v2_6_0/resources/oozie-default.xml") HIVE_DEFAULT = x.load_hadoop_xml_defaults("plugins/vanilla/v2_6_0/resources/hive-default.xml") XML_CONFS = { "Hadoop": [CORE_DEFAULT], "HDFS": [HDFS_DEFAULT], "YARN": [YARN_DEFAULT],
from sahara.plugins import utils from sahara.plugins.vanilla import utils as vu from sahara.plugins.vanilla.v1_2_1 import mysql_helper as m_h from sahara.plugins.vanilla.v1_2_1 import oozie_helper as o_h from sahara.swift import swift_helper as swift from sahara.topology import topology_helper as topology from sahara.utils import crypto from sahara.utils import types as types from sahara.utils import xmlutils as x conductor = c.API LOG = logging.getLogger(__name__) CONF = cfg.CONF CORE_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/core-default.xml') HDFS_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/hdfs-default.xml') MAPRED_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/mapred-default.xml') HIVE_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/hive-default.xml') # Append Oozie configs fore core-site.xml CORE_DEFAULT += o_h.OOZIE_CORE_DEFAULT XML_CONFS = { "HDFS": [CORE_DEFAULT, HDFS_DEFAULT],
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from oslo_config import cfg import six from sahara.plugins import provisioning as p from sahara.plugins.sandbox.hadoop2 import config_helper as c_helper from sahara.utils import xmlutils as x CONF = cfg.CONF CONF.import_opt("enable_data_locality", "sahara.topology.topology_helper") CORE_DEFAULT = x.load_hadoop_xml_defaults( # a list of a 'property' in ***.xml 'plugins/sandbox/v2_7_1/resources/core-default.xml') HDFS_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/sandbox/v2_7_1/resources/hdfs-default.xml') MAPRED_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/sandbox/v2_7_1/resources/mapred-default.xml') YARN_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/sandbox/v2_7_1/resources/yarn-default.xml') OOZIE_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/sandbox/v2_7_1/resources/oozie-default.xml') HIVE_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/sandbox/v2_7_1/resources/hive-default.xml')
# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from sahara.utils import xmlutils as x OOZIE_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/oozie-default.xml') OOZIE_CORE_DEFAULT = [{ 'name': 'hadoop.proxyuser.hadoop.hosts', 'value': "localhost" }, { 'name': 'hadoop.proxyuser.hadoop.groups', 'value': 'hadoop' }] OOZIE_HEAPSIZE_DEFAULT = "CATALINA_OPTS -Xmx1024m" def get_oozie_required_xml_configs(): """Following configs differ from default configs in oozie-default.xml.""" return {
def get_possible_mapreduce_configs(): '''return a list of possible configuration values for map reduce jobs.''' cfg = xmlutils.load_hadoop_xml_defaults( 'service/edp/resources/mapred-job-config.xml') return cfg
# limitations under the License. from oslo.config import cfg from sahara import exceptions as ex from sahara.openstack.common import log as logging from sahara.plugins import provisioning as p from sahara.utils import types as types from sahara.utils import xmlutils as x CONF = cfg.CONF CONF.import_opt("enable_data_locality", "sahara.topology.topology_helper") LOG = logging.getLogger(__name__) CORE_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v2_3_0/resources/core-default.xml') HDFS_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v2_3_0/resources/hdfs-default.xml') MAPRED_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v2_3_0/resources/mapred-default.xml') YARN_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v2_3_0/resources/yarn-default.xml') OOZIE_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v2_3_0/resources/oozie-default.xml') XML_CONFS = { "Hadoop": [CORE_DEFAULT],
from sahara import conductor as c from sahara.i18n import _ from sahara.i18n import _LI from sahara.openstack.common import log as logging from sahara.plugins import provisioning as p from sahara.plugins import utils from sahara.topology import topology_helper as topology from sahara.utils import types as types from sahara.utils import xmlutils as x conductor = c.API LOG = logging.getLogger(__name__) CONF = cfg.CONF CORE_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/spark/resources/core-default.xml') HDFS_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/spark/resources/hdfs-default.xml') XML_CONFS = {"HDFS": [CORE_DEFAULT, HDFS_DEFAULT]} SPARK_CONFS = { 'Spark': { "OPTIONS": [{ 'name': 'Master port', 'description': 'Start the master on a different port' ' (default: 7077)', 'default': '7077', 'priority': 2, }, {
# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from sahara.plugins import provisioning as p from sahara.utils import xmlutils as x CORE_DEFAULT = x.load_hadoop_xml_defaults("plugins/intel/v3_0_2/resources/hadoop-default.xml") HDFS_DEFAULT = x.load_hadoop_xml_defaults("plugins/intel/v3_0_2/resources/hdfs-default.xml") YARN_DEFAULT = x.load_hadoop_xml_defaults("plugins/intel/v3_0_2/resources/yarn-default.xml") OOZIE_DEFAULT = x.load_hadoop_xml_defaults("plugins/intel/v3_0_2/resources/oozie-default.xml") XML_CONFS = {"Hadoop": [CORE_DEFAULT], "HDFS": [HDFS_DEFAULT], "YARN": [YARN_DEFAULT], "JobFlow": [OOZIE_DEFAULT]} IDH_TARBALL_URL = p.Config( "IDH tarball URL", "general", "cluster", priority=1,
# See the License for the specific language governing permissions and # limitations under the License. from sahara.plugins import provisioning as p from sahara.utils import xmlutils as x CORE_DEFAULT = x.load_hadoop_xml_defaults_with_type_and_locale( 'plugins/intel/v2_5_1/resources/hadoop-default.xml') HDFS_DEFAULT = x.load_hadoop_xml_defaults_with_type_and_locale( 'plugins/intel/v2_5_1/resources/hdfs-default.xml') MAPRED_DEFAULT = x.load_hadoop_xml_defaults_with_type_and_locale( 'plugins/intel/v2_5_1/resources/mapred-default.xml') OOZIE_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/intel/v2_5_1/resources/oozie-default.xml') XML_CONFS = { "Hadoop": [CORE_DEFAULT], "HDFS": [HDFS_DEFAULT], "MapReduce": [MAPRED_DEFAULT], "JobFlow": [OOZIE_DEFAULT] } IDH_TARBALL_URL = p.Config('IDH tarball URL', 'general', 'cluster', priority=1, default_value='http://repo2.intelhadoop.com/' 'setup/setup-intelhadoop-' '2.5.1-en-evaluation.RHEL.tar.gz')
from sahara.plugins import provisioning as p from sahara.plugins import utils from sahara.plugins.vanilla import utils as vu from sahara.plugins.vanilla.v1_2_1 import mysql_helper as m_h from sahara.plugins.vanilla.v1_2_1 import oozie_helper as o_h from sahara.swift import swift_helper as swift from sahara.topology import topology_helper as topology from sahara.utils import crypto from sahara.utils import types as types from sahara.utils import xmlutils as x conductor = c.API LOG = logging.getLogger(__name__) CONF = cfg.CONF CORE_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/core-default.xml') HDFS_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/hdfs-default.xml') MAPRED_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/mapred-default.xml') HIVE_DEFAULT = x.load_hadoop_xml_defaults( 'plugins/vanilla/v1_2_1/resources/hive-default.xml') # Append Oozie configs fore core-site.xml CORE_DEFAULT += o_h.OOZIE_CORE_DEFAULT XML_CONFS = { "HDFS": [CORE_DEFAULT, HDFS_DEFAULT],
def get_possible_mapreduce_configs(): '''return a list of possible configuration values for map reduce jobs.''' cfg = xmlutils.load_hadoop_xml_defaults( 'service/edp/resources/mapred-job-config.xml') return cfg
def load_hadoop_xml_defaults(file_name, package, **kwargs): return xmlutils.load_hadoop_xml_defaults(file_name, package)
def read_default_swift_configs(): return x.load_hadoop_xml_defaults('swift/resources/conf-template.xml')