def test_get_instance_params_default(self): instance_configs = configs.get_instance_params(self.instance) expected = [ { "hdfs-site": { "dfs.datanode.data.dir": "/data1/hdfs/data,/data2/hdfs/data", "dfs.journalnode.edits.dir": "/data1/hdfs/journalnode,/data2/hdfs/journalnode", "dfs.namenode.checkpoint.dir": "/data1/hdfs/namesecondary,/data2/hdfs/namesecondary", "dfs.namenode.name.dir": "/data1/hdfs/namenode,/data2/hdfs/namenode" } }, { "yarn-site": { "yarn.nodemanager.local-dirs": "/data1/yarn/local,/data2/yarn/local", "yarn.nodemanager.log-dirs": "/data1/yarn/log,/data2/yarn/log", "yarn.timeline-service.leveldb-timeline-store.path": "/data1/yarn/timeline,/data2/yarn/timeline" } }, { "oozie-site": { "oozie.service.AuthorizationService.security.enabled": "false" } } ] self.assertConfigEqual(expected, instance_configs)
def test_get_instance_params_default(self): instance_configs = configs.get_instance_params(self.instance) expected = [{ "hdfs-site": { "dfs.datanode.data.dir": "/data1/hdfs/data,/data2/hdfs/data", "dfs.journalnode.edits.dir": "/data1/hdfs/journalnode,/data2/hdfs/journalnode", "dfs.namenode.checkpoint.dir": "/data1/hdfs/namesecondary,/data2/hdfs/namesecondary", "dfs.namenode.name.dir": "/data1/hdfs/namenode,/data2/hdfs/namenode" } }, { "yarn-site": { "yarn.nodemanager.local-dirs": "/data1/yarn/local,/data2/yarn/local", "yarn.nodemanager.log-dirs": "/data1/yarn/log,/data2/yarn/log", "yarn.timeline-service.leveldb-timeline-store.path": "/data1/yarn/timeline,/data2/yarn/timeline" } }, { "oozie-site": { "oozie.service.AuthorizationService.security.enabled": "false" } }] self.assertConfigEqual(expected, instance_configs)
def create_blueprint(cluster): _prepare_ranger(cluster) cluster = conductor.cluster_get(context.ctx(), cluster.id) host_groups = [] for ng in cluster.node_groups: procs = p_common.get_ambari_proc_list(ng) procs.extend(p_common.get_clients(cluster)) for instance in ng.instances: hg = { "name": instance.instance_name, "configurations": configs.get_instance_params(instance), "components": [] } for proc in procs: hg["components"].append({"name": proc}) host_groups.append(hg) bp = { "Blueprints": { "stack_name": "HDP", "stack_version": cluster.hadoop_version }, "host_groups": host_groups, "configurations": configs.get_cluster_params(cluster) } ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) password = cluster.extra["ambari_password"] with ambari_client.AmbariClient(ambari, password=password) as client: client.create_blueprint(cluster.name, bp)
def create_blueprint(cluster): _prepare_ranger(cluster) cluster = conductor.cluster_get(context.ctx(), cluster.id) host_groups = [] for ng in cluster.node_groups: procs = p_common.get_ambari_proc_list(ng) procs.extend(p_common.get_clients(cluster)) for instance in ng.instances: hg = { "name": instance.instance_name, "configurations": configs.get_instance_params(instance), "components": get_host_group_components(cluster, procs) } host_groups.append(hg) bp = { "Blueprints": { "stack_name": "HDP", "stack_version": cluster.hadoop_version, }, "host_groups": host_groups, "configurations": configs.get_cluster_params(cluster) } if kerberos.is_kerberos_security_enabled(cluster): bp['configurations'].extend([ _serialize_mit_kdc_kerberos_env(cluster), _serialize_krb5_configs(cluster) ]) bp['Blueprints']['security'] = {'type': 'KERBEROS'} general_configs = cluster.cluster_configs.get("general", {}) if (general_configs.get(p_common.NAMENODE_HA) or general_configs.get(p_common.RESOURCEMANAGER_HA) or general_configs.get(p_common.HBASE_REGIONSERVER_HA)): bp = ha_helper.update_bp_ha_common(cluster, bp) if general_configs.get(p_common.NAMENODE_HA): bp = ha_helper.update_bp_for_namenode_ha(cluster, bp) if general_configs.get(p_common.RESOURCEMANAGER_HA): bp = ha_helper.update_bp_for_resourcemanager_ha(cluster, bp) if general_configs.get(p_common.HBASE_REGIONSERVER_HA): bp = ha_helper.update_bp_for_hbase_ha(cluster, bp) with _get_ambari_client(cluster) as client: return client.create_blueprint(cluster.name, bp)
def test_get_instance_params(self): self.ng.node_configs = { "YARN": { "mapreduce.map.java.opts": "-Dk=v", "yarn.scheduler.minimum-allocation-mb": "256" } } instance_configs = configs.get_instance_params(self.instance) expected = [ { "hdfs-site": { "dfs.datanode.data.dir": "/data1/hdfs/data,/data2/hdfs/data", "dfs.journalnode.edits.dir": "/data1/hdfs/journalnode,/data2/hdfs/journalnode", "dfs.namenode.checkpoint.dir": "/data1/hdfs/namesecondary,/data2/hdfs/namesecondary", "dfs.namenode.name.dir": "/data1/hdfs/namenode,/data2/hdfs/namenode" } }, { "mapred-site": { "mapreduce.map.java.opts": "-Dk=v" } }, { "yarn-site": { "yarn.nodemanager.local-dirs": "/data1/yarn/local,/data2/yarn/local", "yarn.nodemanager.log-dirs": "/data1/yarn/log,/data2/yarn/log", "yarn.scheduler.minimum-allocation-mb": "256", "yarn.timeline-service.leveldb-timeline-store.path": "/data1/yarn/timeline,/data2/yarn/timeline" } }, { "oozie-site": { "oozie.service.AuthorizationService.security.enabled": "false" } } ] self.assertConfigEqual(expected, instance_configs)
def create_blueprint(cluster): _prepare_ranger(cluster) cluster = conductor.cluster_get(context.ctx(), cluster.id) host_groups = [] for ng in cluster.node_groups: procs = p_common.get_ambari_proc_list(ng) procs.extend(p_common.get_clients(cluster)) for instance in ng.instances: hg = { "name": instance.instance_name, "configurations": configs.get_instance_params(instance), "components": [] } for proc in procs: hg["components"].append({"name": proc}) host_groups.append(hg) bp = { "Blueprints": { "stack_name": "HDP", "stack_version": cluster.hadoop_version }, "host_groups": host_groups, "configurations": configs.get_cluster_params(cluster) } ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) password = cluster.extra["ambari_password"] general_configs = cluster.cluster_configs.get("general", {}) if (general_configs.get(p_common.NAMENODE_HA) or general_configs.get(p_common.RESOURCEMANAGER_HA) or general_configs.get(p_common.HBASE_REGIONSERVER_HA)): bp = ha_helper.update_bp_ha_common(cluster, bp) if general_configs.get(p_common.NAMENODE_HA): bp = ha_helper.update_bp_for_namenode_ha(cluster, bp) if general_configs.get(p_common.RESOURCEMANAGER_HA): bp = ha_helper.update_bp_for_resourcemanager_ha(cluster, bp) if general_configs.get(p_common.HBASE_REGIONSERVER_HA): bp = ha_helper.update_bp_for_hbase_ha(cluster, bp) with ambari_client.AmbariClient(ambari, password=password) as client: return client.create_blueprint(cluster.name, bp)
def test_get_instance_params(self): self.ng.node_configs = { "YARN": { "mapreduce.map.java.opts": "-Dk=v", "yarn.scheduler.minimum-allocation-mb": "256" } } instance_configs = configs.get_instance_params(self.instance) expected = [{ "hdfs-site": { "dfs.datanode.data.dir": "/data1/hdfs/data,/data2/hdfs/data", "dfs.journalnode.edits.dir": "/data1/hdfs/journalnode,/data2/hdfs/journalnode", "dfs.namenode.checkpoint.dir": "/data1/hdfs/namesecondary,/data2/hdfs/namesecondary", "dfs.namenode.name.dir": "/data1/hdfs/namenode,/data2/hdfs/namenode" } }, { "mapred-site": { "mapreduce.map.java.opts": "-Dk=v" } }, { "yarn-site": { "yarn.nodemanager.local-dirs": "/data1/yarn/local,/data2/yarn/local", "yarn.nodemanager.log-dirs": "/data1/yarn/log,/data2/yarn/log", "yarn.scheduler.minimum-allocation-mb": "256", "yarn.timeline-service.leveldb-timeline-store.path": "/data1/yarn/timeline,/data2/yarn/timeline" } }, { "oozie-site": { "oozie.service.AuthorizationService.security.enabled": "false" } }] self.assertConfigEqual(expected, instance_configs)