def create_blueprint(cluster): _prepare_ranger(cluster) cluster = conductor.cluster_get(context.ctx(), cluster.id) host_groups = [] for ng in cluster.node_groups: procs = p_common.get_ambari_proc_list(ng) procs.extend(p_common.get_clients(cluster)) for instance in ng.instances: hg = { "name": instance.instance_name, "configurations": configs.get_instance_params(instance), "components": [] } for proc in procs: hg["components"].append({"name": proc}) host_groups.append(hg) bp = { "Blueprints": { "stack_name": "HDP", "stack_version": cluster.hadoop_version }, "host_groups": host_groups, "configurations": configs.get_cluster_params(cluster) } ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) password = cluster.extra["ambari_password"] with ambari_client.AmbariClient(ambari, password=password) as client: client.create_blueprint(cluster.name, bp)
def scale_cluster(self, cluster, instances): deploy.setup_agents(cluster, instances) cluster = conductor.cluster_get(context.ctx(), cluster.id) deploy.wait_host_registration(cluster, instances) deploy.add_new_hosts(cluster, instances) deploy.manage_config_groups(cluster, instances) deploy.manage_host_components(cluster, instances) swift_helper.install_ssl_certs(instances)
def _setup_instances(self, cluster, instances): if (CONF.use_identity_api_v3 and vu.get_hiveserver(cluster) and c_helper.is_swift_enable(cluster)): cluster = proxy.create_proxy_user_for_cluster(cluster) instances = utils.get_instances(cluster) extra = self._extract_configs_to_extra(cluster) cluster = conductor.cluster_get(context.ctx(), cluster) self._push_configs_to_nodes(cluster, extra, instances)
def configure_cluster(self, cluster): deploy.setup_ambari(cluster) deploy.setup_agents(cluster) deploy.wait_ambari_accessible(cluster) deploy.update_default_ambari_password(cluster) cluster = conductor.cluster_get(context.ctx(), cluster.id) deploy.wait_host_registration(cluster) deploy.set_up_hdp_repos(cluster) deploy.create_blueprint(cluster)
def _setup_instances(self, cluster, instances): if (CONF.use_identity_api_v3 and CONF.use_domain_for_proxy_users and vu.get_hiveserver(cluster) and c_helper.is_swift_enable(cluster)): cluster = proxy.create_proxy_user_for_cluster(cluster) instances = utils.get_instances(cluster) extra = self._extract_configs_to_extra(cluster) cluster = conductor.cluster_get(context.ctx(), cluster) self._push_configs_to_nodes(cluster, extra, instances)
def update_default_ambari_password(cluster): ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) new_password = uuidutils.generate_uuid() with ambari_client.AmbariClient(ambari) as client: client.update_user_password("admin", "admin", new_password) extra = cluster.extra.to_dict() if cluster.extra else {} extra["ambari_password"] = new_password ctx = context.ctx() conductor.cluster_update(ctx, cluster, {"extra": extra}) cluster = conductor.cluster_get(ctx, cluster.id)
def configure_cluster(self, cluster): deploy.disable_repos(cluster) deploy.setup_ambari(cluster) deploy.setup_agents(cluster) deploy.wait_ambari_accessible(cluster) deploy.update_default_ambari_password(cluster) cluster = conductor.cluster_get(context.ctx(), cluster.id) deploy.wait_host_registration(cluster) deploy.set_up_hdp_repos(cluster) deploy.create_blueprint(cluster)
def get_sentry_db_password(cluster): ctx = context.ctx() cluster = conductor.cluster_get(ctx, cluster.id) passwd = cluster.extra.get('sentry_db_password') if cluster.extra else None if passwd: return passwd passwd = six.text_type(uuid.uuid4()) extra = cluster.extra.to_dict() if cluster.extra else {} extra['sentry_db_password'] = passwd cluster = conductor.cluster_update(ctx, cluster, {'extra': extra}) return passwd
def get_password_from_db(cluster, pwname): ctx = context.ctx() cluster = conductor.cluster_get(ctx, cluster.id) passwd = cluster.extra.get(pwname) if cluster.extra else None if passwd: return passwd passwd = six.text_type(uuid.uuid4()) extra = cluster.extra.to_dict() if cluster.extra else {} extra[pwname] = passwd cluster = conductor.cluster_update(ctx, cluster, {'extra': extra}) return passwd
def validate_creation(cluster_id): ctx = context.ctx() cluster = conductor.cluster_get(ctx, cluster_id) _check_ambari(cluster) _check_hdfs(cluster) _check_yarn(cluster) _check_oozie(cluster) _check_hive(cluster) _check_hbase(cluster) _check_spark(cluster) _check_ranger(cluster) _check_storm(cluster)
def scale_cluster(self, cluster, instances): deploy.prepare_kerberos(cluster, instances) deploy.setup_agents(cluster, instances) cluster = conductor.cluster_get(context.ctx(), cluster.id) deploy.wait_host_registration(cluster, instances) deploy.resolve_package_conflicts(cluster, instances) deploy.add_new_hosts(cluster, instances) deploy.manage_config_groups(cluster, instances) deploy.manage_host_components(cluster, instances) deploy.configure_rack_awareness(cluster, instances) swift_helper.install_ssl_certs(instances) deploy.add_hadoop_swift_jar(instances) deploy.deploy_kerberos_principals(cluster, instances)
def configure_cluster(self, cluster): deploy.disable_repos(cluster) deploy.setup_ambari(cluster) deploy.setup_agents(cluster) deploy.wait_ambari_accessible(cluster) deploy.update_default_ambari_password(cluster) cluster = conductor.cluster_get(context.ctx(), cluster.id) deploy.wait_host_registration(cluster, plugin_utils.get_instances(cluster)) deploy.prepare_kerberos(cluster) deploy.set_up_hdp_repos(cluster) deploy.resolve_package_conflicts(cluster) deploy.create_blueprint(cluster)
def delete_password(cluster, pw_name): """delete the named password from the key manager This function will lookup the named password in the cluster entry and delete it from the key manager. :param cluster: The cluster record containing the password :param pw_name: The name associated with the password """ ctx = context.ctx() cluster = conductor.cluster_get(ctx, cluster.id) key_id = cluster.extra.get(pw_name) if cluster.extra else None if key_id is not None: key_manager.delete_key(key_id, ctx)
def _set_cluster_info(self, cluster): ambari_ip = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER).management_ip ambari_port = "8080" info = { p_common.AMBARI_SERVER: { "Web UI": "http://{host}:{port}".format(host=ambari_ip, port=ambari_port), "Username": "******", "Password": cluster.extra["ambari_password"], } } namenode = plugin_utils.get_instance(cluster, p_common.NAMENODE) if namenode: info[p_common.NAMENODE] = {"Web UI": "http://%s:50070" % namenode.management_ip} resourcemanager = plugin_utils.get_instance(cluster, p_common.RESOURCEMANAGER) if resourcemanager: info[p_common.RESOURCEMANAGER] = {"Web UI": "http://%s:8088" % resourcemanager.management_ip} historyserver = plugin_utils.get_instance(cluster, p_common.HISTORYSERVER) if historyserver: info[p_common.HISTORYSERVER] = {"Web UI": "http://%s:19888" % historyserver.management_ip} atlserver = plugin_utils.get_instance(cluster, p_common.APP_TIMELINE_SERVER) if atlserver: info[p_common.APP_TIMELINE_SERVER] = {"Web UI": "http://%s:8188" % atlserver.management_ip} oozie = plugin_utils.get_instance(cluster, p_common.OOZIE_SERVER) if oozie: info[p_common.OOZIE_SERVER] = {"Web UI": "http://%s:11000/oozie" % oozie.management_ip} hbase_master = plugin_utils.get_instance(cluster, p_common.HBASE_MASTER) if hbase_master: info[p_common.HBASE_MASTER] = {"Web UI": "http://%s:60010" % hbase_master.management_ip} falcon = plugin_utils.get_instance(cluster, p_common.FALCON_SERVER) if falcon: info[p_common.FALCON_SERVER] = {"Web UI": "http://%s:15000" % falcon.management_ip} storm_ui = plugin_utils.get_instance(cluster, p_common.STORM_UI_SERVER) if storm_ui: info[p_common.STORM_UI_SERVER] = {"Web UI": "http://%s:8744" % storm_ui.management_ip} ranger_admin = plugin_utils.get_instance(cluster, p_common.RANGER_ADMIN) if ranger_admin: info[p_common.RANGER_ADMIN] = { "Web UI": "http://%s:6080" % ranger_admin.management_ip, "Username": "******", "Password": "******", } spark_hs = plugin_utils.get_instance(cluster, p_common.SPARK_JOBHISTORYSERVER) if spark_hs: info[p_common.SPARK_JOBHISTORYSERVER] = {"Web UI": "http://%s:18080" % spark_hs.management_ip} info.update(cluster.info.to_dict()) ctx = context.ctx() conductor.cluster_update(ctx, cluster, {"info": info}) cluster = conductor.cluster_get(ctx, cluster.id)
def create_blueprint(cluster): _prepare_ranger(cluster) cluster = conductor.cluster_get(context.ctx(), cluster.id) host_groups = [] for ng in cluster.node_groups: procs = p_common.get_ambari_proc_list(ng) procs.extend(p_common.get_clients(cluster)) for instance in ng.instances: hg = { "name": instance.instance_name, "configurations": configs.get_instance_params(instance), "components": get_host_group_components(cluster, procs) } host_groups.append(hg) bp = { "Blueprints": { "stack_name": "HDP", "stack_version": cluster.hadoop_version, }, "host_groups": host_groups, "configurations": configs.get_cluster_params(cluster) } if kerberos.is_kerberos_security_enabled(cluster): bp['configurations'].extend([ _serialize_mit_kdc_kerberos_env(cluster), _serialize_krb5_configs(cluster) ]) bp['Blueprints']['security'] = {'type': 'KERBEROS'} general_configs = cluster.cluster_configs.get("general", {}) if (general_configs.get(p_common.NAMENODE_HA) or general_configs.get(p_common.RESOURCEMANAGER_HA) or general_configs.get(p_common.HBASE_REGIONSERVER_HA)): bp = ha_helper.update_bp_ha_common(cluster, bp) if general_configs.get(p_common.NAMENODE_HA): bp = ha_helper.update_bp_for_namenode_ha(cluster, bp) if general_configs.get(p_common.RESOURCEMANAGER_HA): bp = ha_helper.update_bp_for_resourcemanager_ha(cluster, bp) if general_configs.get(p_common.HBASE_REGIONSERVER_HA): bp = ha_helper.update_bp_for_hbase_ha(cluster, bp) with _get_ambari_client(cluster) as client: return client.create_blueprint(cluster.name, bp)
def create_blueprint(cluster): _prepare_ranger(cluster) cluster = conductor.cluster_get(context.ctx(), cluster.id) host_groups = [] for ng in cluster.node_groups: procs = p_common.get_ambari_proc_list(ng) procs.extend(p_common.get_clients(cluster)) for instance in ng.instances: hg = { "name": instance.instance_name, "configurations": configs.get_instance_params(instance), "components": [] } for proc in procs: hg["components"].append({"name": proc}) host_groups.append(hg) bp = { "Blueprints": { "stack_name": "HDP", "stack_version": cluster.hadoop_version }, "host_groups": host_groups, "configurations": configs.get_cluster_params(cluster) } ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) password = cluster.extra["ambari_password"] general_configs = cluster.cluster_configs.get("general", {}) if (general_configs.get(p_common.NAMENODE_HA) or general_configs.get(p_common.RESOURCEMANAGER_HA) or general_configs.get(p_common.HBASE_REGIONSERVER_HA)): bp = ha_helper.update_bp_ha_common(cluster, bp) if general_configs.get(p_common.NAMENODE_HA): bp = ha_helper.update_bp_for_namenode_ha(cluster, bp) if general_configs.get(p_common.RESOURCEMANAGER_HA): bp = ha_helper.update_bp_for_resourcemanager_ha(cluster, bp) if general_configs.get(p_common.HBASE_REGIONSERVER_HA): bp = ha_helper.update_bp_for_hbase_ha(cluster, bp) with ambari_client.AmbariClient(ambari, password=password) as client: return client.create_blueprint(cluster.name, bp)
def get_password(cluster, pw_name): """return a password for the named entry This function will return, or create and return, a password for the named entry. It will store the password in the key manager and use the ID in the database entry. :param cluster: The cluster record containing the password :param pw_name: The entry name associated with the password :returns: The cleartext password """ ctx = context.ctx() cluster = conductor.cluster_get(ctx, cluster.id) passwd = cluster.extra.get(pw_name) if cluster.extra else None if passwd: return key_manager.get_secret(passwd, ctx) passwd = six.text_type(uuid.uuid4()) extra = cluster.extra.to_dict() if cluster.extra else {} extra[pw_name] = key_manager.store_secret(passwd, ctx) cluster = conductor.cluster_update(ctx, cluster, {'extra': extra}) return passwd
def _setup_instances(self, cluster, instances): extra = self._extract_configs_to_extra(cluster) cluster = conductor.cluster_get(context.ctx(), cluster) self._push_configs_to_nodes(cluster, extra, instances)
def _set_cluster_info(self, cluster): ambari_ip = plugin_utils.get_instance( cluster, p_common.AMBARI_SERVER).get_ip_or_dns_name() ambari_port = "8080" info = { p_common.AMBARI_SERVER: { "Web UI": "http://{host}:{port}".format(host=ambari_ip, port=ambari_port), "Username": "******", "Password": cluster.extra["ambari_password"] } } nns = plugin_utils.get_instances(cluster, p_common.NAMENODE) info[p_common.NAMENODE] = {} for idx, namenode in enumerate(nns): info[p_common.NAMENODE]["Web UI %s" % (idx + 1)] = ( "http://%s:50070" % namenode.get_ip_or_dns_name()) rms = plugin_utils.get_instances(cluster, p_common.RESOURCEMANAGER) info[p_common.RESOURCEMANAGER] = {} for idx, resourcemanager in enumerate(rms): info[p_common.RESOURCEMANAGER]["Web UI %s" % (idx + 1)] = ( "http://%s:8088" % resourcemanager.get_ip_or_dns_name()) historyserver = plugin_utils.get_instance(cluster, p_common.HISTORYSERVER) if historyserver: info[p_common.HISTORYSERVER] = { "Web UI": "http://%s:19888" % historyserver.get_ip_or_dns_name() } atlserver = plugin_utils.get_instance(cluster, p_common.APP_TIMELINE_SERVER) if atlserver: info[p_common.APP_TIMELINE_SERVER] = { "Web UI": "http://%s:8188" % atlserver.get_ip_or_dns_name() } oozie = plugin_utils.get_instance(cluster, p_common.OOZIE_SERVER) if oozie: info[p_common.OOZIE_SERVER] = { "Web UI": "http://%s:11000/oozie" % oozie.get_ip_or_dns_name() } hbase_master = plugin_utils.get_instance(cluster, p_common.HBASE_MASTER) if hbase_master: info[p_common.HBASE_MASTER] = { "Web UI": "http://%s:60010" % hbase_master.get_ip_or_dns_name() } falcon = plugin_utils.get_instance(cluster, p_common.FALCON_SERVER) if falcon: info[p_common.FALCON_SERVER] = { "Web UI": "http://%s:15000" % falcon.get_ip_or_dns_name() } storm_ui = plugin_utils.get_instance(cluster, p_common.STORM_UI_SERVER) if storm_ui: info[p_common.STORM_UI_SERVER] = { "Web UI": "http://%s:8744" % storm_ui.get_ip_or_dns_name() } ranger_admin = plugin_utils.get_instance(cluster, p_common.RANGER_ADMIN) if ranger_admin: info[p_common.RANGER_ADMIN] = { "Web UI": "http://%s:6080" % ranger_admin.get_ip_or_dns_name(), "Username": "******", "Password": "******" } spark_hs = plugin_utils.get_instance(cluster, p_common.SPARK_JOBHISTORYSERVER) if spark_hs: info[p_common.SPARK_JOBHISTORYSERVER] = { "Web UI": "http://%s:18080" % spark_hs.get_ip_or_dns_name() } info.update(cluster.info.to_dict()) ctx = context.ctx() conductor.cluster_update(ctx, cluster, {"info": info}) cluster = conductor.cluster_get(ctx, cluster.id)
def cluster_get(context, cluster_id, **kwargs): return conductor.cluster_get(context, cluster_id)