def create_blueprint(cluster):
_prepare_ranger(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
host_groups = []
for ng in cluster.node_groups:
procs = p_common.get_ambari_proc_list(ng)
procs.extend(p_common.get_clients(cluster))
for instance in ng.instances:
hg = {
"name": instance.instance_name,
"configurations": configs.get_instance_params(instance),
"components": []
}
for proc in procs:
hg["components"].append({"name": proc})
host_groups.append(hg)
bp = {
"Blueprints": {
"stack_name": "HDP",
"stack_version": cluster.hadoop_version
},
"host_groups": host_groups,
"configurations": configs.get_cluster_params(cluster)
}
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
password = cluster.extra["ambari_password"]
with ambari_client.AmbariClient(ambari, password=password) as client:
client.create_blueprint(cluster.name, bp)
def create_blueprint(cluster):
_prepare_ranger(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
host_groups = []
for ng in cluster.node_groups:
procs = p_common.get_ambari_proc_list(ng)
procs.extend(p_common.get_clients(cluster))
for instance in ng.instances:
hg = {
"name": instance.instance_name,
"configurations": configs.get_instance_params(instance),
"components": []
}
for proc in procs:
hg["components"].append({"name": proc})
host_groups.append(hg)
bp = {
"Blueprints": {
"stack_name": "HDP",
"stack_version": cluster.hadoop_version
},
"host_groups": host_groups,
"configurations": configs.get_cluster_params(cluster)
}
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
password = cluster.extra["ambari_password"]
with ambari_client.AmbariClient(ambari, password=password) as client:
client.create_blueprint(cluster.name, bp)
def scale_cluster(self, cluster, instances):
deploy.setup_agents(cluster, instances)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
deploy.wait_host_registration(cluster, instances)
deploy.add_new_hosts(cluster, instances)
deploy.manage_config_groups(cluster, instances)
deploy.manage_host_components(cluster, instances)
swift_helper.install_ssl_certs(instances)
def scale_cluster(self, cluster, instances):
deploy.setup_agents(cluster, instances)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
deploy.wait_host_registration(cluster, instances)
deploy.add_new_hosts(cluster, instances)
deploy.manage_config_groups(cluster, instances)
deploy.manage_host_components(cluster, instances)
swift_helper.install_ssl_certs(instances)
def _setup_instances(self, cluster, instances):
if (CONF.use_identity_api_v3 and vu.get_hiveserver(cluster)
and c_helper.is_swift_enable(cluster)):
cluster = proxy.create_proxy_user_for_cluster(cluster)
instances = utils.get_instances(cluster)
extra = self._extract_configs_to_extra(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster)
self._push_configs_to_nodes(cluster, extra, instances)
def configure_cluster(self, cluster):
deploy.setup_ambari(cluster)
deploy.setup_agents(cluster)
deploy.wait_ambari_accessible(cluster)
deploy.update_default_ambari_password(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
deploy.wait_host_registration(cluster)
deploy.set_up_hdp_repos(cluster)
deploy.create_blueprint(cluster)
def _setup_instances(self, cluster, instances):
if (CONF.use_identity_api_v3 and CONF.use_domain_for_proxy_users and
vu.get_hiveserver(cluster) and
c_helper.is_swift_enable(cluster)):
cluster = proxy.create_proxy_user_for_cluster(cluster)
instances = utils.get_instances(cluster)
extra = self._extract_configs_to_extra(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster)
self._push_configs_to_nodes(cluster, extra, instances)
def update_default_ambari_password(cluster):
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
new_password = uuidutils.generate_uuid()
with ambari_client.AmbariClient(ambari) as client:
client.update_user_password("admin", "admin", new_password)
extra = cluster.extra.to_dict() if cluster.extra else {}
extra["ambari_password"] = new_password
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {"extra": extra})
cluster = conductor.cluster_get(ctx, cluster.id)
def update_default_ambari_password(cluster):
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
new_password = uuidutils.generate_uuid()
with ambari_client.AmbariClient(ambari) as client:
client.update_user_password("admin", "admin", new_password)
extra = cluster.extra.to_dict() if cluster.extra else {}
extra["ambari_password"] = new_password
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {"extra": extra})
cluster = conductor.cluster_get(ctx, cluster.id)
def configure_cluster(self, cluster):
deploy.disable_repos(cluster)
deploy.setup_ambari(cluster)
deploy.setup_agents(cluster)
deploy.wait_ambari_accessible(cluster)
deploy.update_default_ambari_password(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
deploy.wait_host_registration(cluster)
deploy.set_up_hdp_repos(cluster)
deploy.create_blueprint(cluster)
def get_sentry_db_password(cluster):
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster.id)
passwd = cluster.extra.get('sentry_db_password') if cluster.extra else None
if passwd:
return passwd
passwd = six.text_type(uuid.uuid4())
extra = cluster.extra.to_dict() if cluster.extra else {}
extra['sentry_db_password'] = passwd
cluster = conductor.cluster_update(ctx, cluster, {'extra': extra})
return passwd
def get_password_from_db(cluster, pwname):
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster.id)
passwd = cluster.extra.get(pwname) if cluster.extra else None
if passwd:
return passwd
passwd = six.text_type(uuid.uuid4())
extra = cluster.extra.to_dict() if cluster.extra else {}
extra[pwname] = passwd
cluster = conductor.cluster_update(ctx, cluster, {'extra': extra})
return passwd
def validate_creation(cluster_id):
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster_id)
_check_ambari(cluster)
_check_hdfs(cluster)
_check_yarn(cluster)
_check_oozie(cluster)
_check_hive(cluster)
_check_hbase(cluster)
_check_spark(cluster)
_check_ranger(cluster)
_check_storm(cluster)
def validate_creation(cluster_id):
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster_id)
_check_ambari(cluster)
_check_hdfs(cluster)
_check_yarn(cluster)
_check_oozie(cluster)
_check_hive(cluster)
_check_hbase(cluster)
_check_spark(cluster)
_check_ranger(cluster)
_check_storm(cluster)
def scale_cluster(self, cluster, instances):
deploy.prepare_kerberos(cluster, instances)
deploy.setup_agents(cluster, instances)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
deploy.wait_host_registration(cluster, instances)
deploy.resolve_package_conflicts(cluster, instances)
deploy.add_new_hosts(cluster, instances)
deploy.manage_config_groups(cluster, instances)
deploy.manage_host_components(cluster, instances)
deploy.configure_rack_awareness(cluster, instances)
swift_helper.install_ssl_certs(instances)
deploy.add_hadoop_swift_jar(instances)
deploy.deploy_kerberos_principals(cluster, instances)
def configure_cluster(self, cluster):
deploy.disable_repos(cluster)
deploy.setup_ambari(cluster)
deploy.setup_agents(cluster)
deploy.wait_ambari_accessible(cluster)
deploy.update_default_ambari_password(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
deploy.wait_host_registration(cluster,
plugin_utils.get_instances(cluster))
deploy.prepare_kerberos(cluster)
deploy.set_up_hdp_repos(cluster)
deploy.resolve_package_conflicts(cluster)
deploy.create_blueprint(cluster)
def delete_password(cluster, pw_name):
"""delete the named password from the key manager
This function will lookup the named password in the cluster entry
and delete it from the key manager.
:param cluster: The cluster record containing the password
:param pw_name: The name associated with the password
"""
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster.id)
key_id = cluster.extra.get(pw_name) if cluster.extra else None
if key_id is not None:
key_manager.delete_key(key_id, ctx)
def delete_password(cluster, pw_name):
"""delete the named password from the key manager
This function will lookup the named password in the cluster entry
and delete it from the key manager.
:param cluster: The cluster record containing the password
:param pw_name: The name associated with the password
"""
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster.id)
key_id = cluster.extra.get(pw_name) if cluster.extra else None
if key_id is not None:
key_manager.delete_key(key_id, ctx)
def _set_cluster_info(self, cluster):
ambari_ip = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER).management_ip
ambari_port = "8080"
info = {
p_common.AMBARI_SERVER: {
"Web UI": "http://{host}:{port}".format(host=ambari_ip, port=ambari_port),
"Username": "******",
"Password": cluster.extra["ambari_password"],
}
}
namenode = plugin_utils.get_instance(cluster, p_common.NAMENODE)
if namenode:
info[p_common.NAMENODE] = {"Web UI": "http://%s:50070" % namenode.management_ip}
resourcemanager = plugin_utils.get_instance(cluster, p_common.RESOURCEMANAGER)
if resourcemanager:
info[p_common.RESOURCEMANAGER] = {"Web UI": "http://%s:8088" % resourcemanager.management_ip}
historyserver = plugin_utils.get_instance(cluster, p_common.HISTORYSERVER)
if historyserver:
info[p_common.HISTORYSERVER] = {"Web UI": "http://%s:19888" % historyserver.management_ip}
atlserver = plugin_utils.get_instance(cluster, p_common.APP_TIMELINE_SERVER)
if atlserver:
info[p_common.APP_TIMELINE_SERVER] = {"Web UI": "http://%s:8188" % atlserver.management_ip}
oozie = plugin_utils.get_instance(cluster, p_common.OOZIE_SERVER)
if oozie:
info[p_common.OOZIE_SERVER] = {"Web UI": "http://%s:11000/oozie" % oozie.management_ip}
hbase_master = plugin_utils.get_instance(cluster, p_common.HBASE_MASTER)
if hbase_master:
info[p_common.HBASE_MASTER] = {"Web UI": "http://%s:60010" % hbase_master.management_ip}
falcon = plugin_utils.get_instance(cluster, p_common.FALCON_SERVER)
if falcon:
info[p_common.FALCON_SERVER] = {"Web UI": "http://%s:15000" % falcon.management_ip}
storm_ui = plugin_utils.get_instance(cluster, p_common.STORM_UI_SERVER)
if storm_ui:
info[p_common.STORM_UI_SERVER] = {"Web UI": "http://%s:8744" % storm_ui.management_ip}
ranger_admin = plugin_utils.get_instance(cluster, p_common.RANGER_ADMIN)
if ranger_admin:
info[p_common.RANGER_ADMIN] = {
"Web UI": "http://%s:6080" % ranger_admin.management_ip,
"Username": "******",
"Password": "******",
}
spark_hs = plugin_utils.get_instance(cluster, p_common.SPARK_JOBHISTORYSERVER)
if spark_hs:
info[p_common.SPARK_JOBHISTORYSERVER] = {"Web UI": "http://%s:18080" % spark_hs.management_ip}
info.update(cluster.info.to_dict())
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {"info": info})
cluster = conductor.cluster_get(ctx, cluster.id)
def create_blueprint(cluster):
_prepare_ranger(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
host_groups = []
for ng in cluster.node_groups:
procs = p_common.get_ambari_proc_list(ng)
procs.extend(p_common.get_clients(cluster))
for instance in ng.instances:
hg = {
"name": instance.instance_name,
"configurations": configs.get_instance_params(instance),
"components": get_host_group_components(cluster, procs)
}
host_groups.append(hg)
bp = {
"Blueprints": {
"stack_name": "HDP",
"stack_version": cluster.hadoop_version,
},
"host_groups": host_groups,
"configurations": configs.get_cluster_params(cluster)
}
if kerberos.is_kerberos_security_enabled(cluster):
bp['configurations'].extend([
_serialize_mit_kdc_kerberos_env(cluster),
_serialize_krb5_configs(cluster)
])
bp['Blueprints']['security'] = {'type': 'KERBEROS'}
general_configs = cluster.cluster_configs.get("general", {})
if (general_configs.get(p_common.NAMENODE_HA)
or general_configs.get(p_common.RESOURCEMANAGER_HA)
or general_configs.get(p_common.HBASE_REGIONSERVER_HA)):
bp = ha_helper.update_bp_ha_common(cluster, bp)
if general_configs.get(p_common.NAMENODE_HA):
bp = ha_helper.update_bp_for_namenode_ha(cluster, bp)
if general_configs.get(p_common.RESOURCEMANAGER_HA):
bp = ha_helper.update_bp_for_resourcemanager_ha(cluster, bp)
if general_configs.get(p_common.HBASE_REGIONSERVER_HA):
bp = ha_helper.update_bp_for_hbase_ha(cluster, bp)
with _get_ambari_client(cluster) as client:
return client.create_blueprint(cluster.name, bp)
def create_blueprint(cluster):
_prepare_ranger(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
host_groups = []
for ng in cluster.node_groups:
procs = p_common.get_ambari_proc_list(ng)
procs.extend(p_common.get_clients(cluster))
for instance in ng.instances:
hg = {
"name": instance.instance_name,
"configurations": configs.get_instance_params(instance),
"components": []
}
for proc in procs:
hg["components"].append({"name": proc})
host_groups.append(hg)
bp = {
"Blueprints": {
"stack_name": "HDP",
"stack_version": cluster.hadoop_version
},
"host_groups": host_groups,
"configurations": configs.get_cluster_params(cluster)
}
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
password = cluster.extra["ambari_password"]
general_configs = cluster.cluster_configs.get("general", {})
if (general_configs.get(p_common.NAMENODE_HA)
or general_configs.get(p_common.RESOURCEMANAGER_HA)
or general_configs.get(p_common.HBASE_REGIONSERVER_HA)):
bp = ha_helper.update_bp_ha_common(cluster, bp)
if general_configs.get(p_common.NAMENODE_HA):
bp = ha_helper.update_bp_for_namenode_ha(cluster, bp)
if general_configs.get(p_common.RESOURCEMANAGER_HA):
bp = ha_helper.update_bp_for_resourcemanager_ha(cluster, bp)
if general_configs.get(p_common.HBASE_REGIONSERVER_HA):
bp = ha_helper.update_bp_for_hbase_ha(cluster, bp)
with ambari_client.AmbariClient(ambari, password=password) as client:
return client.create_blueprint(cluster.name, bp)
def create_blueprint(cluster):
_prepare_ranger(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
host_groups = []
for ng in cluster.node_groups:
procs = p_common.get_ambari_proc_list(ng)
procs.extend(p_common.get_clients(cluster))
for instance in ng.instances:
hg = {
"name": instance.instance_name,
"configurations": configs.get_instance_params(instance),
"components": []
}
for proc in procs:
hg["components"].append({"name": proc})
host_groups.append(hg)
bp = {
"Blueprints": {
"stack_name": "HDP",
"stack_version": cluster.hadoop_version
},
"host_groups": host_groups,
"configurations": configs.get_cluster_params(cluster)
}
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
password = cluster.extra["ambari_password"]
general_configs = cluster.cluster_configs.get("general", {})
if (general_configs.get(p_common.NAMENODE_HA) or
general_configs.get(p_common.RESOURCEMANAGER_HA) or
general_configs.get(p_common.HBASE_REGIONSERVER_HA)):
bp = ha_helper.update_bp_ha_common(cluster, bp)
if general_configs.get(p_common.NAMENODE_HA):
bp = ha_helper.update_bp_for_namenode_ha(cluster, bp)
if general_configs.get(p_common.RESOURCEMANAGER_HA):
bp = ha_helper.update_bp_for_resourcemanager_ha(cluster, bp)
if general_configs.get(p_common.HBASE_REGIONSERVER_HA):
bp = ha_helper.update_bp_for_hbase_ha(cluster, bp)
with ambari_client.AmbariClient(ambari, password=password) as client:
return client.create_blueprint(cluster.name, bp)
def get_password(cluster, pw_name):
"""return a password for the named entry
This function will return, or create and return, a password for the
named entry. It will store the password in the key manager and use
the ID in the database entry.
:param cluster: The cluster record containing the password
:param pw_name: The entry name associated with the password
:returns: The cleartext password
"""
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster.id)
passwd = cluster.extra.get(pw_name) if cluster.extra else None
if passwd:
return key_manager.get_secret(passwd, ctx)
passwd = six.text_type(uuid.uuid4())
extra = cluster.extra.to_dict() if cluster.extra else {}
extra[pw_name] = key_manager.store_secret(passwd, ctx)
cluster = conductor.cluster_update(ctx, cluster, {'extra': extra})
return passwd
def get_password(cluster, pw_name):
"""return a password for the named entry
This function will return, or create and return, a password for the
named entry. It will store the password in the key manager and use
the ID in the database entry.
:param cluster: The cluster record containing the password
:param pw_name: The entry name associated with the password
:returns: The cleartext password
"""
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster.id)
passwd = cluster.extra.get(pw_name) if cluster.extra else None
if passwd:
return key_manager.get_secret(passwd, ctx)
passwd = six.text_type(uuid.uuid4())
extra = cluster.extra.to_dict() if cluster.extra else {}
extra[pw_name] = key_manager.store_secret(passwd, ctx)
cluster = conductor.cluster_update(ctx, cluster, {'extra': extra})
return passwd
def _setup_instances(self, cluster, instances):
extra = self._extract_configs_to_extra(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster)
self._push_configs_to_nodes(cluster, extra, instances)
def _setup_instances(self, cluster, instances):
extra = self._extract_configs_to_extra(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster)
self._push_configs_to_nodes(cluster, extra, instances)
def _set_cluster_info(self, cluster):
ambari_ip = plugin_utils.get_instance(
cluster, p_common.AMBARI_SERVER).get_ip_or_dns_name()
ambari_port = "8080"
info = {
p_common.AMBARI_SERVER: {
"Web UI":
"http://{host}:{port}".format(host=ambari_ip,
port=ambari_port),
"Username":
"******",
"Password":
cluster.extra["ambari_password"]
}
}
nns = plugin_utils.get_instances(cluster, p_common.NAMENODE)
info[p_common.NAMENODE] = {}
for idx, namenode in enumerate(nns):
info[p_common.NAMENODE]["Web UI %s" % (idx + 1)] = (
"http://%s:50070" % namenode.get_ip_or_dns_name())
rms = plugin_utils.get_instances(cluster, p_common.RESOURCEMANAGER)
info[p_common.RESOURCEMANAGER] = {}
for idx, resourcemanager in enumerate(rms):
info[p_common.RESOURCEMANAGER]["Web UI %s" % (idx + 1)] = (
"http://%s:8088" % resourcemanager.get_ip_or_dns_name())
historyserver = plugin_utils.get_instance(cluster,
p_common.HISTORYSERVER)
if historyserver:
info[p_common.HISTORYSERVER] = {
"Web UI":
"http://%s:19888" % historyserver.get_ip_or_dns_name()
}
atlserver = plugin_utils.get_instance(cluster,
p_common.APP_TIMELINE_SERVER)
if atlserver:
info[p_common.APP_TIMELINE_SERVER] = {
"Web UI": "http://%s:8188" % atlserver.get_ip_or_dns_name()
}
oozie = plugin_utils.get_instance(cluster, p_common.OOZIE_SERVER)
if oozie:
info[p_common.OOZIE_SERVER] = {
"Web UI": "http://%s:11000/oozie" % oozie.get_ip_or_dns_name()
}
hbase_master = plugin_utils.get_instance(cluster,
p_common.HBASE_MASTER)
if hbase_master:
info[p_common.HBASE_MASTER] = {
"Web UI":
"http://%s:60010" % hbase_master.get_ip_or_dns_name()
}
falcon = plugin_utils.get_instance(cluster, p_common.FALCON_SERVER)
if falcon:
info[p_common.FALCON_SERVER] = {
"Web UI": "http://%s:15000" % falcon.get_ip_or_dns_name()
}
storm_ui = plugin_utils.get_instance(cluster, p_common.STORM_UI_SERVER)
if storm_ui:
info[p_common.STORM_UI_SERVER] = {
"Web UI": "http://%s:8744" % storm_ui.get_ip_or_dns_name()
}
ranger_admin = plugin_utils.get_instance(cluster,
p_common.RANGER_ADMIN)
if ranger_admin:
info[p_common.RANGER_ADMIN] = {
"Web UI": "http://%s:6080" % ranger_admin.get_ip_or_dns_name(),
"Username": "******",
"Password": "******"
}
spark_hs = plugin_utils.get_instance(cluster,
p_common.SPARK_JOBHISTORYSERVER)
if spark_hs:
info[p_common.SPARK_JOBHISTORYSERVER] = {
"Web UI": "http://%s:18080" % spark_hs.get_ip_or_dns_name()
}
info.update(cluster.info.to_dict())
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {"info": info})
cluster = conductor.cluster_get(ctx, cluster.id)
def cluster_get(context, cluster_id, **kwargs):
return conductor.cluster_get(context, cluster_id)
