def get_open_ports(self, node_group): cluster = node_group.cluster ports_map = { 'namenode': [8020, 50070, 50470], 'datanode': [50010, 1004, 50075, 1006, 50020], 'master': [ int(utils.get_config_value_or_default("Spark", "Master port", cluster)), int(utils.get_config_value_or_default("Spark", "Master webui port", cluster)), ], 'slave': [ int(utils.get_config_value_or_default("Spark", "Worker webui port", cluster)) ], 'zeppelin': [int(utils.get_config_value_or_default("Zeppelin", "Web UI port", cluster))] } ports = [] for process in node_group.node_processes: if process in ports_map: ports.extend(ports_map[process]) return ports
def __init__(self, cluster): super(EdpEngine, self).__init__(cluster) self.master = plugin_utils.get_instance(cluster, "master") self.plugin_params["spark-user"] = "" self.plugin_params["spark-submit"] = os.path.join( plugin_utils.get_config_value_or_default("Spark", "Spark home", self.cluster), "bin/spark-submit" ) self.plugin_params["deploy-mode"] = "client" port_str = six.text_type(plugin_utils.get_config_value_or_default("Spark", "Master port", self.cluster)) self.plugin_params["master"] = "spark://%(host)s:" + port_str driver_cp = plugin_utils.get_config_value_or_default("Spark", "Executor extra classpath", self.cluster) self.plugin_params["driver-class-path"] = driver_cp
def generate_zk_basic_config(cluster): args = { 'ticktime': utils.get_config_value_or_default("ZooKeeper", "tickTime", cluster), 'initlimit': utils.get_config_value_or_default("ZooKeeper", "initLimit", cluster), 'synclimit': utils.get_config_value_or_default("ZooKeeper", "syncLimit", cluster) } zoo_cfg = f.get_file_text( 'plugins/vanilla/hadoop2/resources/zoo_sample.cfg') return zoo_cfg.format(**args)
def __init__(self, cluster): super(EdpSparkEngine, self).__init__(cluster) self.master = plugin_utils.get_instance(cluster, "spark history server") self.plugin_params["spark-user"] = "******" self.plugin_params["spark-submit"] = os.path.join( plugin_utils.get_config_value_or_default("Spark", "Spark home", self.cluster), "bin/spark-submit") self.plugin_params["deploy-mode"] = "cluster" self.plugin_params["master"] = "yarn" driver_cp = plugin_utils.get_config_value_or_default( "Spark", "Executor extra classpath", self.cluster) self.plugin_params["driver-class-path"] = driver_cp
def generate_spark_executor_classpath(cluster): cp = utils.get_config_value_or_default("Spark", "Executor extra classpath", cluster) if cp: return "spark.executor.extraClassPath " + cp return "\n"
def _validate_existing_ng_scaling(self, cluster, existing): scalable_processes = self._get_scalable_processes() dn_to_delete = 0 for ng in cluster.node_groups: if ng.id in existing: if ng.count > existing[ng.id] and ("datanode" in ng.node_processes): dn_to_delete += ng.count - existing[ng.id] if not set(ng.node_processes).issubset(scalable_processes): raise ex.NodeGroupCannotBeScaled( ng.name, _("Spark plugin cannot scale nodegroup" " with processes: %s") % ' '.join(ng.node_processes)) dn_amount = len(utils.get_instances(cluster, "datanode")) rep_factor = utils.get_config_value_or_default('HDFS', "dfs.replication", cluster) if dn_to_delete > 0 and dn_amount - dn_to_delete < rep_factor: raise ex.ClusterCannotBeScaled( cluster.name, _("Spark plugin cannot shrink cluster because " "there would be not enough nodes for HDFS " "replicas (replication factor is %s)") % rep_factor)
def validate(self, cluster): if cluster.hadoop_version == "1.0.0": raise exceptions.DeprecatedException( _("Support for Spark version 1.0.0 is now deprecated and will" " be removed in the 2016.1 release.") ) nn_count = sum([ng.count for ng in utils.get_node_groups(cluster, "namenode")]) if nn_count != 1: raise ex.InvalidComponentCountException("namenode", 1, nn_count) dn_count = sum([ng.count for ng in utils.get_node_groups(cluster, "datanode")]) if dn_count < 1: raise ex.InvalidComponentCountException("datanode", _("1 or more"), nn_count) rep_factor = utils.get_config_value_or_default("HDFS", "dfs.replication", cluster) if dn_count < rep_factor: raise ex.InvalidComponentCountException( "datanode", _("%s or more") % rep_factor, dn_count, _("Number of %(dn)s instances should not be less " "than %(replication)s") % {"dn": "datanode", "replication": "dfs.replication"}, ) # validate Spark Master Node and Spark Slaves sm_count = sum([ng.count for ng in utils.get_node_groups(cluster, "master")]) if sm_count != 1: raise ex.RequiredServiceMissingException("Spark master") sl_count = sum([ng.count for ng in utils.get_node_groups(cluster, "slave")]) if sl_count < 1: raise ex.InvalidComponentCountException("Spark slave", _("1 or more"), sl_count)
def __init__(self, cluster): super(EdpEngine, self).__init__(cluster) self.master = plugin_utils.get_instance(cluster, "master") self.plugin_params["spark-user"] = "" self.plugin_params["spark-submit"] = os.path.join( plugin_utils. get_config_value_or_default("Spark", "Spark home", self.cluster), "bin/spark-submit") self.plugin_params["deploy-mode"] = "client" port_str = six.text_type( plugin_utils.get_config_value_or_default( "Spark", "Master port", self.cluster)) self.plugin_params["master"] = ('spark://%(host)s:' + port_str) driver_cp = plugin_utils.get_config_value_or_default( "Spark", "Executor extra classpath", self.cluster) self.plugin_params["driver-class-path"] = driver_cp
def _set_cluster_info(self, cluster): nn = utils.get_instance(cluster, "namenode") sp_master = utils.get_instance(cluster, "master") info = {} if nn: address = utils.get_config_value_or_default("HDFS", "dfs.http.address", cluster) port = address[address.rfind(":") + 1 :] info["HDFS"] = {"Web UI": "http://%s:%s" % (nn.management_ip, port)} info["HDFS"]["NameNode"] = "hdfs://%s:8020" % nn.hostname() if sp_master: port = utils.get_config_value_or_default("Spark", "Master webui port", cluster) if port is not None: info["Spark"] = {"Web UI": "http://%s:%s" % (sp_master.management_ip, port)} ctx = context.ctx() conductor.cluster_update(ctx, cluster, {"info": info})
def __init__(self, cluster): super(EdpSparkEngine, self).__init__(cluster) self.master = u.get_instance(cluster, "SPARK_YARN_HISTORY_SERVER") self.plugin_params["spark-user"] = "******" self.plugin_params["spark-submit"] = "spark-submit" self.plugin_params["deploy-mode"] = "cluster" self.plugin_params["master"] = "yarn-cluster" driver_cp = u.get_config_value_or_default("Spark", "Executor extra classpath", self.cluster) self.plugin_params["driver-class-path"] = driver_cp
def generate_spark_env_configs(cluster): configs = [] # master configuration sp_master = utils.get_instance(cluster, "master") configs.append('SPARK_MASTER_IP=' + sp_master.hostname()) # point to the hadoop conf dir so that Spark can read things # like the swift configuration without having to copy core-site # to /opt/spark/conf configs.append('HADOOP_CONF_DIR=' + HADOOP_CONF_DIR) masterport = utils.get_config_value_or_default("Spark", "Master port", cluster) if masterport and masterport != _get_spark_opt_default("Master port"): configs.append('SPARK_MASTER_PORT=' + str(masterport)) masterwebport = utils.get_config_value_or_default("Spark", "Master webui port", cluster) if (masterwebport and masterwebport != _get_spark_opt_default("Master webui port")): configs.append('SPARK_MASTER_WEBUI_PORT=' + str(masterwebport)) # configuration for workers workercores = utils.get_config_value_or_default("Spark", "Worker cores", cluster) if workercores and workercores != _get_spark_opt_default("Worker cores"): configs.append('SPARK_WORKER_CORES=' + str(workercores)) workermemory = utils.get_config_value_or_default("Spark", "Worker memory", cluster) if (workermemory and workermemory != _get_spark_opt_default("Worker memory")): configs.append('SPARK_WORKER_MEMORY=' + str(workermemory)) workerport = utils.get_config_value_or_default("Spark", "Worker port", cluster) if workerport and workerport != _get_spark_opt_default("Worker port"): configs.append('SPARK_WORKER_PORT=' + str(workerport)) workerwebport = utils.get_config_value_or_default("Spark", "Worker webui port", cluster) if (workerwebport and workerwebport != _get_spark_opt_default("Worker webui port")): configs.append('SPARK_WORKER_WEBUI_PORT=' + str(workerwebport)) workerinstances = utils.get_config_value_or_default("Spark", "Worker instances", cluster) if (workerinstances and workerinstances != _get_spark_opt_default("Worker instances")): configs.append('SPARK_WORKER_INSTANCES=' + str(workerinstances)) return '\n'.join(configs)
def generate_job_cleanup_config(cluster): args = { 'minimum_cleanup_megabytes': utils.get_config_value_or_default( "Spark", "Minimum cleanup megabytes", cluster), 'minimum_cleanup_seconds': utils.get_config_value_or_default( "Spark", "Minimum cleanup seconds", cluster), 'maximum_cleanup_seconds': utils.get_config_value_or_default( "Spark", "Maximum cleanup seconds", cluster) } job_conf = {'valid': (args['maximum_cleanup_seconds'] > 0 and (args['minimum_cleanup_megabytes'] > 0 and args['minimum_cleanup_seconds'] > 0))} if job_conf['valid']: job_conf['cron'] = f.get_file_text( 'plugins/spark/resources/spark-cleanup.cron'), job_cleanup_script = f.get_file_text( 'plugins/spark/resources/tmp-cleanup.sh.template') job_conf['script'] = job_cleanup_script.format(**args) return job_conf
def __init__(self, cluster): super(EdpSparkEngine, self).__init__(cluster) self.master = u.get_instance(cluster, "SPARK_YARN_HISTORY_SERVER") self.plugin_params["spark-user"] = "******" self.plugin_params["spark-submit"] = "spark-submit" self.plugin_params["deploy-mode"] = "cluster" self.plugin_params["master"] = "yarn-cluster" driver_cp = u.get_config_value_or_default( "Spark", "Executor extra classpath", self.cluster) self.plugin_params["driver-class-path"] = driver_cp
def __init__(self, cluster): super(EdpSparkEngine, self).__init__(cluster) self.master = u.get_instance(cluster, "CLOUDERA_MANAGER") self.plugin_params["spark-user"] = "******" self.plugin_params["spark-submit"] = "spark-submit" self.plugin_params["deploy-mode"] = "cluster" self.plugin_params["master"] = "yarn-cluster" driver_cp = u.get_config_value_or_default( "Spark", "Executor extra classpath", self.cluster) if driver_cp: driver_cp = " --driver-class-path " + driver_cp self.plugin_params["driver-class-path"] = driver_cp
def generate_zeppelin_setup_script(sp_master): script_lines = ["#!/bin/bash -x"] script_lines.append( "echo 'export MASTER=spark://{0}:{1}' " ">> /opt/incubator-zeppelin/conf/zeppelin-env.sh".format( sp_master['instance_name'], utils.get_config_value_or_default( "Spark", "Master port", sp_master.node_group.cluster))) script_lines.append("echo 'export SPARK_HOME=/opt/spark' >> " "/opt/incubator-zeppelin/conf/zeppelin-env.sh") script_lines.append("echo 'export PYTHONPATH=$SPARK_HOME/python:" "$SPARK_HOME/python/lib/py4j-0.8.2.1-src.zip:" "$PYTHONPATH' >> " "/opt/incubator-zeppelin/conf/zeppelin-env.sh") script_lines.append("sed -i 's|<value>8080</value>|<value>{0}</value>|g'" " /opt/incubator-zeppelin/conf/zeppelin-site.xml" .format(utils.get_config_value_or_default( "Zeppelin", "Web UI port", sp_master.node_group.cluster))) return "\n".join(script_lines)
def __init__(self, cluster): super(EdpSparkEngine, self).__init__(cluster) self.master = u.get_instance(cluster, "CLOUDERA_MANAGER") self.plugin_params["spark-user"] = "******" self.plugin_params["spark-submit"] = "spark-submit" self.plugin_params["deploy-mode"] = "cluster" self.plugin_params["master"] = "yarn-cluster" driver_cp = u.get_config_value_or_default("Spark", "Executor extra classpath", self.cluster) if driver_cp: driver_cp = " --driver-class-path " + driver_cp self.plugin_params["driver-class-path"] = driver_cp
def _set_cluster_info(self, cluster): nn = utils.get_instance(cluster, "namenode") sp_master = utils.get_instance(cluster, "master") info = {} if nn: address = utils.get_config_value_or_default( 'HDFS', 'dfs.http.address', cluster) port = address[address.rfind(':') + 1:] info['HDFS'] = { 'Web UI': 'http://%s:%s' % (nn.management_ip, port) } info['HDFS']['NameNode'] = 'hdfs://%s:8020' % nn.hostname() if sp_master: port = utils.get_config_value_or_default( 'Spark', 'Master webui port', cluster) if port is not None: info['Spark'] = { 'Web UI': 'http://%s:%s' % (sp_master.management_ip, port) } ctx = context.ctx() conductor.cluster_update(ctx, cluster, {'info': info})
def _set_cluster_info(self, cluster): nn = utils.get_instance(cluster, "namenode") sp_master = utils.get_instance(cluster, "master") info = {} if nn: address = utils.get_config_value_or_default( 'HDFS', 'dfs.http.address', cluster) port = address[address.rfind(':') + 1:] info['HDFS'] = { 'Web UI': 'http://%s:%s' % (nn.management_ip, port) } info['HDFS']['NameNode'] = 'hdfs://%s:8020' % nn.hostname() if sp_master: port = utils.get_config_value_or_default('Spark', 'Master webui port', cluster) if port is not None: info['Spark'] = { 'Web UI': 'http://%s:%s' % (sp_master.management_ip, port) } ctx = context.ctx() conductor.cluster_update(ctx, cluster, {'info': info})
def validate(self, cluster): if cluster.hadoop_version == "1.0.0": raise exceptions.DeprecatedException( _("Support for Spark version 1.0.0 is now deprecated and will" " be removed in the 2016.1 release.")) nn_count = sum( [ng.count for ng in utils.get_node_groups(cluster, "namenode")]) if nn_count != 1: raise ex.InvalidComponentCountException("namenode", 1, nn_count) dn_count = sum( [ng.count for ng in utils.get_node_groups(cluster, "datanode")]) if dn_count < 1: raise ex.InvalidComponentCountException("datanode", _("1 or more"), nn_count) rep_factor = utils.get_config_value_or_default('HDFS', "dfs.replication", cluster) if dn_count < rep_factor: raise ex.InvalidComponentCountException( 'datanode', _('%s or more') % rep_factor, dn_count, _('Number of %(dn)s instances should not be less ' 'than %(replication)s') % { 'dn': 'datanode', 'replication': 'dfs.replication' }) # validate Spark Master Node and Spark Slaves sm_count = sum( [ng.count for ng in utils.get_node_groups(cluster, "master")]) if sm_count != 1: raise ex.RequiredServiceMissingException("Spark master") sl_count = sum( [ng.count for ng in utils.get_node_groups(cluster, "slave")]) if sl_count < 1: raise ex.InvalidComponentCountException("Spark slave", _("1 or more"), sl_count)
def validate(self, cluster): nn_count = sum( [ng.count for ng in utils.get_node_groups(cluster, "namenode")]) if nn_count != 1: raise ex.InvalidComponentCountException("namenode", 1, nn_count) dn_count = sum( [ng.count for ng in utils.get_node_groups(cluster, "datanode")]) if dn_count < 1: raise ex.InvalidComponentCountException("datanode", _("1 or more"), nn_count) rep_factor = utils.get_config_value_or_default('HDFS', "dfs.replication", cluster) if dn_count < rep_factor: raise ex.InvalidComponentCountException( 'datanode', _('%s or more') % rep_factor, dn_count, _('Number of %(dn)s instances should not be less ' 'than %(replication)s') % { 'dn': 'datanode', 'replication': 'dfs.replication' }) # validate Spark Master Node and Spark Slaves sm_count = sum( [ng.count for ng in utils.get_node_groups(cluster, "master")]) if sm_count < 1: raise ex.RequiredServiceMissingException("Spark master") if sm_count >= 2: raise ex.InvalidComponentCountException("Spark master", "1", sm_count) sl_count = sum( [ng.count for ng in utils.get_node_groups(cluster, "slave")]) if sl_count < 1: raise ex.InvalidComponentCountException("Spark slave", _("1 or more"), sl_count)
def decommission_sl(master, inst_to_be_deleted, survived_inst): if survived_inst is not None: slavenames = [] for slave in survived_inst: slavenames.append(slave.hostname()) slaves_content = c_helper.generate_spark_slaves_configs(slavenames) else: slaves_content = "\n" cluster = master.cluster sp_home = utils.get_config_value_or_default("Spark", "Spark home", cluster) r_master = remote.get_remote(master) run.stop_spark(r_master, sp_home) # write new slave file to master files = {os.path.join(sp_home, 'conf/slaves'): slaves_content} r_master.write_files_to(files) # write new slaves file to each survived slave as well for i in survived_inst: with remote.get_remote(i) as r: r.write_files_to(files) run.start_spark_master(r_master, sp_home)
def decommission_sl(master, inst_to_be_deleted, survived_inst): if survived_inst is not None: slavenames = [] for slave in survived_inst: slavenames.append(slave.hostname()) slaves_content = c_helper.generate_spark_slaves_configs(slavenames) else: slaves_content = "\n" cluster = master.cluster sp_home = utils.get_config_value_or_default("Spark", "Spark home", cluster) r_master = utils.get_remote(master) run.stop_spark(r_master, sp_home) # write new slave file to master files = {os.path.join(sp_home, 'conf/slaves'): slaves_content} r_master.write_files_to(files) # write new slaves file to each survived slave as well for i in survived_inst: with utils.get_remote(i) as r: r.write_files_to(files) run.start_spark_master(r_master, sp_home)
def validate(self, cluster): nn_count = sum([ng.count for ng in utils.get_node_groups(cluster, "namenode")]) if nn_count != 1: raise ex.InvalidComponentCountException("namenode", 1, nn_count) dn_count = sum([ng.count for ng in utils.get_node_groups(cluster, "datanode")]) if dn_count < 1: raise ex.InvalidComponentCountException("datanode", _("1 or more"), nn_count) rep_factor = utils.get_config_value_or_default('HDFS', "dfs.replication", cluster) if dn_count < rep_factor: raise ex.InvalidComponentCountException( 'datanode', _('%s or more') % rep_factor, dn_count, _('Number of %(dn)s instances should not be less ' 'than %(replication)s') % {'dn': 'datanode', 'replication': 'dfs.replication'}) # validate Spark Master Node and Spark Slaves sm_count = sum([ng.count for ng in utils.get_node_groups(cluster, "master")]) if sm_count != 1: raise ex.RequiredServiceMissingException("Spark master") sl_count = sum([ng.count for ng in utils.get_node_groups(cluster, "slave")]) if sl_count < 1: raise ex.InvalidComponentCountException("Spark slave", _("1 or more"), sl_count)
def get_admin_password(cluster): # TODO(vgridnev): support in follow-up improved secret storage for # configs return pl_utils.get_config_value_or_default( cluster=cluster, config=admin_password)
def get_policy_url(cluster): return pl_utils.get_config_value_or_default( cluster=cluster, config=policy_url)
def get_realm_name(cluster): return pl_utils.get_config_value_or_default( cluster=cluster, config=realm_name)
def get_admin_principal(cluster): return pl_utils.get_config_value_or_default( cluster=cluster, config=admin_principal)
def is_kerberos_security_enabled(cluster): return pl_utils.get_config_value_or_default(cluster=cluster, config=enable_kerberos)
def _spark_home(self, cluster): return utils.get_config_value_or_default("Spark", "Spark home", cluster)
def get_realm_name(cluster): return pl_utils.get_config_value_or_default(cluster=cluster, config=realm_name)
def get_admin_password(cluster): # TODO(vgridnev): support in follow-up improved secret storage for # configs return pl_utils.get_config_value_or_default(cluster=cluster, config=admin_password)
def get_admin_principal(cluster): return pl_utils.get_config_value_or_default(cluster=cluster, config=admin_principal)
def get_policy_url(cluster): return pl_utils.get_config_value_or_default(cluster=cluster, config=policy_url)
def generate_spark_env_configs(cluster): configs = [] # newly added: master configuration sp_master = utils.get_instance(cluster, "master") configs.append('SPARK_MASTER_IP=' + sp_master.hostname()) # point to the hadoop conf dir so that Spark can read things # like the swift configuration without having to copy core-site # to /opt/spark/conf HADOOP_CONF_DIR = '/opt/hadoop/etc/hadoop' configs.append('HADOOP_CONF_DIR=' + HADOOP_CONF_DIR) # 获取Master Port的用户配置信息 masterport = utils.get_config_value_or_default("Spark", "Master port", cluster) if masterport and masterport != _get_spark_opt_default("Master port"): configs.append('SPARK_MASTER_PORT=' + str(masterport)) # 获取Master Web Port的用户配置信息 masterwebport = utils.get_config_value_or_default("Spark", "Master webui port", cluster) if (masterwebport and masterwebport != _get_spark_opt_default("Master webui port")): configs.append('SPARK_MASTER_WEBUI_PORT=' + str(masterwebport)) # work节点的配置信息 workercores = utils.get_config_value_or_default("Spark", "Worker cores", cluster) if workercores and workercores != _get_spark_opt_default("Worker cores"): configs.append('SPARK_WORKER_CORES=' + str(workercores)) workermemory = utils.get_config_value_or_default("Spark", "Worker memory", cluster) if workermemory and workermemory != _get_spark_opt_default( "Worker memory"): configs.append('SPARK_WORKER_MEMORY=' + str(workermemory)) workerport = utils.get_config_value_or_default("Spark", "Worker port", cluster) if workerport and workerport != _get_spark_opt_default("Worker port"): configs.append('SPARK_WORKER_PORT=' + str(workerport)) workerwebport = utils.get_config_value_or_default("Spark", "Worker webui port", cluster) if workerwebport and workerwebport != _get_spark_opt_default( "Worker webui port"): configs.append('SPARK_WORKER_WEBUI_PORT=' + str(workerwebport)) workerinstances = utils.get_config_value_or_default( "Spark", "Worker instances", cluster) if workerinstances and workerinstances != _get_spark_opt_default( "Worker instances"): configs.append('SPARK_WORKER_INSTANCES=' + str(workerinstances)) # Hadoop and YARN configs there are in one folder configs.append('YARN_CONF_DIR=' + HADOOP_CONF_DIR) return '\n'.join(configs)
def test_get_config_value_or_default(self, mock_plugins_base): # no config self.assertRaises(RuntimeError, pu.get_config_value_or_default) config = mock.Mock() config.applicable_target = "service" config.name = "name" config.default_value = "default_value" # cluster has the config cluster = mock.Mock() cluster.cluster_configs = {"service": {"name": "name"}} cluster.plugin_name = "plugin_name" cluster.hadoop_version = "hadoop_version" res = pu.get_config_value_or_default(cluster=cluster, config=config) self.assertEqual("name", res) # node group has the config cluster.cluster_configs = {} node_group1 = mock.Mock() node_group2 = mock.Mock() node_group1.configuration = mock.Mock(return_value={"service": {}}) node_group2.configuration = mock.Mock( return_value={"service": {"name": "name"}}) cluster.node_groups = [node_group1, node_group2] res = pu.get_config_value_or_default(cluster=cluster, config=config) self.assertEqual("name", res) # cluster doesn't have the config, neither the node groups # so it returns the default value cluster.node_groups = [] res = pu.get_config_value_or_default(cluster=cluster, config=config) self.assertEqual("default_value", res) # no config specified, but there's a config for the plugin # with this service and name mock_get_all_configs = mock.Mock(return_value=[config]) mock_plugin = mock.Mock() mock_plugin.get_all_configs = mock_get_all_configs mock_get_plugin = mock.Mock(return_value=mock_plugin) mock_PLUGINS = mock.Mock() mock_PLUGINS.get_plugin = mock_get_plugin mock_plugins_base.PLUGINS = mock_PLUGINS res = pu.get_config_value_or_default(cluster=cluster, service="service", name="name") self.assertEqual("default_value", res) mock_get_plugin.assert_called_once_with("plugin_name") mock_get_all_configs.assert_called_once_with("hadoop_version") # no config especified and no existing config for this plugin # with this service or name cluster.plugin_name = "plugin_name2" cluster.hadoop_version = "hadoop_version2" self.assertRaises(RuntimeError, pu.get_config_value_or_default, cluster=cluster, service="newService", name="name") mock_get_plugin.assert_called_with("plugin_name2") self.assertEqual(2, mock_get_plugin.call_count) mock_get_all_configs.assert_called_with("hadoop_version2") self.assertEqual(2, mock_get_all_configs.call_count)
def is_kerberos_security_enabled(cluster): return pl_utils.get_config_value_or_default( cluster=cluster, config=enable_kerberos)
def get_port_from_config(service, name, cluster=None): address = utils.get_config_value_or_default(service, name, cluster) return utils.get_port_from_address(address)
def get_kdc_server_ip(cluster): return pl_utils.get_config_value_or_default(cluster=cluster, config=kdc_server_ip)
def using_existing_kdc(cluster): return pl_utils.get_config_value_or_default( cluster=cluster, config=use_existing_kdc)
def using_existing_kdc(cluster): return pl_utils.get_config_value_or_default(cluster=cluster, config=use_existing_kdc)
def get_kdc_server_ip(cluster): return pl_utils.get_config_value_or_default( cluster=cluster, config=kdc_server_ip)
def test_get_config_value_or_default(self, mock_plugins_base): # no config self.assertRaises(RuntimeError, pu.get_config_value_or_default) config = mock.Mock() config.applicable_target = "service" config.name = "name" config.default_value = "default_value" # cluster has the config cluster = mock.Mock() cluster.cluster_configs = {"service": {"name": "name"}} cluster.plugin_name = "plugin_name" cluster.hadoop_version = "hadoop_version" res = pu.get_config_value_or_default(cluster=cluster, config=config) self.assertEqual("name", res) # node group has the config cluster.cluster_configs = {} node_group1 = mock.Mock() node_group2 = mock.Mock() node_group1.configuration = mock.Mock(return_value={"service": {}}) node_group2.configuration = mock.Mock( return_value={"service": { "name": "name" }}) cluster.node_groups = [node_group1, node_group2] res = pu.get_config_value_or_default(cluster=cluster, config=config) self.assertEqual("name", res) # cluster doesn't have the config, neither the node groups # so it returns the default value cluster.node_groups = [] res = pu.get_config_value_or_default(cluster=cluster, config=config) self.assertEqual("default_value", res) # no config specified, but there's a config for the plugin # with this service and name mock_get_all_configs = mock.Mock(return_value=[config]) mock_plugin = mock.Mock() mock_plugin.get_all_configs = mock_get_all_configs mock_get_plugin = mock.Mock(return_value=mock_plugin) mock_PLUGINS = mock.Mock() mock_PLUGINS.get_plugin = mock_get_plugin mock_plugins_base.PLUGINS = mock_PLUGINS res = pu.get_config_value_or_default(cluster=cluster, service="service", name="name") self.assertEqual("default_value", res) mock_get_plugin.assert_called_once_with("plugin_name") mock_get_all_configs.assert_called_once_with("hadoop_version") # no config especified and no existing config for this plugin # with this service or name cluster.plugin_name = "plugin_name2" cluster.hadoop_version = "hadoop_version2" self.assertRaises(RuntimeError, pu.get_config_value_or_default, cluster=cluster, service="newService", name="name") mock_get_plugin.assert_called_with("plugin_name2") self.assertEqual(2, mock_get_plugin.call_count) mock_get_all_configs.assert_called_with("hadoop_version2") self.assertEqual(2, mock_get_all_configs.call_count)