def get_hive_password(cluster):
cluster = conductor.cluster_get(context.ctx(), cluster)
extra = cluster.extra.to_dict()
if 'hive_pass_id' not in extra:
extra['hive_pass_id'] = u.generate_random_password()
conductor.cluster_update(context.ctx(), cluster, {'extra': extra})
return castellan.get_secret(extra['hive_pass_id'])
def test_cleanup_configs(self):
remote = mock.Mock()
instance = mock.Mock()
extra_conf = {'job_cleanup': {
'valid': True,
'script': 'script_text',
'cron': 'cron_text'}}
instance.node_group.node_processes = ["master"]
instance.node_group.id = id
cluster_dict = self._init_cluster_dict('2.2')
cluster = conductor.cluster_create(context.ctx(), cluster_dict)
plugin = pb.PLUGINS.get_plugin(cluster.plugin_name)
plugin._push_cleanup_job(remote, cluster, extra_conf, instance)
remote.write_file_to.assert_called_with(
'/etc/hadoop/tmp-cleanup.sh',
'script_text')
remote.execute_command.assert_called_with(
'sudo sh -c \'echo "cron_text" > /etc/cron.d/spark-cleanup\'')
remote.reset_mock()
instance.node_group.node_processes = ["worker"]
plugin._push_cleanup_job(remote, cluster, extra_conf, instance)
self.assertFalse(remote.called)
remote.reset_mock()
instance.node_group.node_processes = ["master"]
extra_conf['job_cleanup']['valid'] = False
plugin._push_cleanup_job(remote, cluster, extra_conf, instance)
remote.execute_command.assert_called_with(
'sudo rm -f /etc/crond.d/spark-cleanup')
def _update_cluster_info(self, cluster_context):
LOG.debug('Updating UI information.')
info = {'Admin user credentials': {'Username': '******',
'Password': pu.get_mapr_password
(cluster_context.cluster)}}
for service in cluster_context.cluster_services:
for title, node_process, ui_info in (
service.get_ui_info(cluster_context)):
removed = cluster_context.removed_instances(node_process)
instances = cluster_context.get_instances(node_process)
instances = [i for i in instances if i not in removed]
if len(instances) == 1:
display_name_template = "%(title)s"
else:
display_name_template = "%(title)s %(index)s"
for index, instance in enumerate(instances, start=1):
args = {"title": title, "index": index}
display_name = display_name_template % args
data = ui_info.copy()
data[srvc.SERVICE_UI] = (data[srvc.SERVICE_UI] %
instance.get_ip_or_dns_name())
info.update({display_name: data})
ctx = context.ctx()
conductor.cluster_update(ctx, cluster_context.cluster, {'info': info})
def _set_cluster_info(self, cluster):
nn = utils.get_instance(cluster, "namenode")
sp_master = utils.get_instance(cluster, "master")
info = {}
if nn:
address = utils.get_config_value_or_default(
'HDFS', 'dfs.http.address', cluster)
port = address[address.rfind(':') + 1:]
info['HDFS'] = {
'Web UI': 'http://%s:%s' % (nn.get_ip_or_dns_name(), port)
}
info['HDFS']['NameNode'] = 'hdfs://%s:8020' % nn.hostname()
if sp_master:
port = utils.get_config_value_or_default('Spark',
'Master webui port',
cluster)
if port is not None:
info['Spark'] = {
'Web UI':
'http://%s:%s' % (sp_master.get_ip_or_dns_name(), port)
}
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {'info': info})
def _prepare_ranger(cluster):
ranger = plugin_utils.get_instance(cluster, p_common.RANGER_ADMIN)
if not ranger:
return
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
with ambari.remote() as r:
sudo = functools.partial(r.execute_command, run_as_root=True)
sudo("ambari-server setup --jdbc-db=mysql "
"--jdbc-driver=/usr/share/java/mysql-connector-java.jar")
init_db_template = (
"create user 'root'@'%' identified by '{password}';\n"
"set password for 'root'@'localhost' = password('{password}');")
password = uuidutils.generate_uuid()
extra = cluster.extra.to_dict() if cluster.extra else {}
extra["ranger_db_password"] = password
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {"extra": extra})
with ranger.remote() as r:
sudo = functools.partial(r.execute_command, run_as_root=True)
# TODO(sreshetnyak): add ubuntu support
sudo("yum install -y mysql-server")
sudo("service mysqld start")
r.write_file_to("/tmp/init.sql",
init_db_template.format(password=password))
sudo("mysql < /tmp/init.sql")
sudo("rm /tmp/init.sql")
def setup_context(self, username="******", tenant_id="tenant_1",
auth_token="test_auth_token", tenant_name='test_tenant',
service_catalog=None, **kwargs):
self.addCleanup(context.set_ctx,
context.ctx() if context.has_ctx() else None)
context.set_ctx(context.PluginsContext(
username=username, tenant_id=tenant_id,
auth_token=auth_token, service_catalog=service_catalog or {},
tenant_name=tenant_name, **kwargs))
def _set_cluster_info(self, cluster):
info = self.cloudera_utils.get_cloudera_manager_info(cluster)
hue = self.cloudera_utils.pu.get_hue(cluster)
if hue:
info['Hue Dashboard'] = {
'Web UI': 'http://%s:8888' % hue.get_ip_or_dns_name()
}
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {'info': info})
def update_default_ambari_password(cluster):
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
new_password = uuidutils.generate_uuid()
with ambari_client.AmbariClient(ambari) as client:
client.update_user_password("admin", "admin", new_password)
extra = cluster.extra.to_dict() if cluster.extra else {}
extra["ambari_password"] = new_password
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {"extra": extra})
cluster = conductor.cluster_get(ctx, cluster.id)
def validate(cluster_id):
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster_id)
_check_ambari(cluster)
_check_hdfs(cluster)
_check_yarn(cluster)
_check_oozie(cluster)
_check_hive(cluster)
_check_hbase(cluster)
_check_spark(cluster)
_check_ranger(cluster)
_check_storm(cluster)
def test_plugin_edp_engine(self):
cluster_dict = {
'name': 'cluster',
'plugin_name': 'cdh',
'hadoop_version': '5.7.0',
'default_image_id': 'image'}
cluster = conductor.cluster_create(context.ctx(), cluster_dict)
plugin = pb.PLUGINS.get_plugin(cluster.plugin_name)
self.assertIsInstance(
plugin.get_edp_engine(cluster, edp.JOB_TYPE_SPARK),
edp.PluginsSparkJobEngine)
def test_validate(self, mock_utils):
cluster_data = self._get_cluster('cluster', '1.1.0')
cluster = conductor.cluster_create(context.ctx(), cluster_data)
plugin = pb.PLUGINS.get_plugin(cluster.plugin_name)
# number of nimbus nodes != 1 should raise an exception
fake_ng = mock.Mock()
fake_ng.count = 0
mock_ng = mock.Mock(return_value=[fake_ng])
mock_utils.get_node_groups = mock_ng
self.assertRaises(ex.RequiredServiceMissingException, plugin.validate,
cluster)
mock_ng.assert_called_once_with(cluster, "nimbus")
fake_ng.count = 2
self.assertRaises(ex.InvalidComponentCountException, plugin.validate,
cluster)
mock_ng.assert_called_with(cluster, "nimbus")
self.assertEqual(2, mock_ng.call_count)
# no supervisor should raise an exception
fake_nimbus = mock.Mock()
fake_nimbus.count = 1
fake_supervisor = mock.Mock()
fake_supervisor.count = 0
mock_ng = mock.Mock(side_effect=[[fake_nimbus], [fake_supervisor]])
mock_utils.get_node_groups = mock_ng
self.assertRaises(ex.InvalidComponentCountException, plugin.validate,
cluster)
mock_ng.assert_any_call(cluster, "nimbus")
mock_ng.assert_any_call(cluster, "supervisor")
self.assertEqual(2, mock_ng.call_count)
# one nimbus and one or more supervisors should not raise an exception
fake_nimbus.count = 1
fake_supervisor.count = 2
mock_ng = mock.Mock(side_effect=[[fake_nimbus], [fake_supervisor]])
mock_utils.get_node_groups = mock_ng
plugin.validate(cluster)
mock_ng.assert_any_call(cluster, "nimbus")
mock_ng.assert_any_call(cluster, "supervisor")
self.assertEqual(2, mock_ng.call_count)
def scale_cluster(self, cluster, instances):
deploy.prepare_kerberos(cluster, instances)
deploy.setup_agents(cluster, instances)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
deploy.wait_host_registration(cluster, instances)
deploy.resolve_package_conflicts(cluster, instances)
deploy.add_new_hosts(cluster, instances)
deploy.manage_config_groups(cluster, instances)
deploy.manage_host_components(cluster, instances)
deploy.configure_rack_awareness(cluster, instances)
swift_helper.install_ssl_certs(instances)
deploy.add_hadoop_swift_jar(instances)
deploy.deploy_kerberos_principals(cluster, instances)
def configure_cluster(self, cluster):
deploy.disable_repos(cluster)
deploy.setup_ambari(cluster)
deploy.setup_agents(cluster)
deploy.wait_ambari_accessible(cluster)
deploy.update_default_ambari_password(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
deploy.wait_host_registration(cluster,
plugin_utils.get_instances(cluster))
deploy.prepare_kerberos(cluster)
deploy.set_up_hdp_repos(cluster)
deploy.resolve_package_conflicts(cluster)
deploy.create_blueprint(cluster)
def _set_cluster_info(self, cluster):
st_master = utils.get_instance(cluster, "nimbus")
info = {}
if st_master:
port = "8080"
info['Strom'] = {
'Web UI':
'http://%s:%s' % (st_master.get_ip_or_dns_name(), port)
}
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {'info': info})
def test_edp_calls_hadoop2_create_dir(self, create_dir):
for version in ['2.7.1']:
cluster_dict = {
'name': 'cluster' + version.replace('.', '_'),
'plugin_name': 'vanilla',
'hadoop_version': version,
'default_image_id': 'image'}
cluster = conductor.cluster_create(context.ctx(), cluster_dict)
plugin = pb.PLUGINS.get_plugin(cluster.plugin_name)
create_dir.reset_mock()
plugin.get_edp_engine(cluster, edp.JOB_TYPE_PIG).create_hdfs_dir(
mock.Mock(), '/tmp')
self.assertEqual(1, create_dir.call_count)
def delete_password_from_keymanager(cluster, pwname):
"""delete the named password from the key manager
This function will lookup the named password in the cluster entry
and delete it from the key manager.
:param cluster: The cluster record containing the password
:param pwname: The name associated with the password
"""
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster.id)
key_id = cluster.extra.get(pwname) if cluster.extra else None
if key_id is not None:
key_manager.delete_key(key_id, ctx)
def _upload_job_files_to_hdfs(self,
where,
job_dir,
job,
configs,
proxy_configs=None):
mains = job.mains or []
libs = job.libs or []
builtin_libs = edp.get_builtin_binaries(job, configs)
uploaded_paths = []
hdfs_user = self.get_hdfs_user()
lib_dir = job_dir + '/lib'
with where.remote() as r:
for m in mains:
path = jb_manager.JOB_BINARIES. \
get_job_binary_by_url(m.url). \
copy_binary_to_cluster(m, proxy_configs=proxy_configs,
remote=r, context=context.ctx())
target = os.path.join(job_dir, m.name)
mfs.copy_from_local(r, path, target, hdfs_user)
uploaded_paths.append(target)
if len(libs) > 0:
self.create_hdfs_dir(r, lib_dir)
for l in libs:
path = jb_manager.JOB_BINARIES. \
get_job_binary_by_url(l.url). \
copy_binary_to_cluster(l, proxy_configs=proxy_configs,
remote=r, context=context.ctx())
target = os.path.join(lib_dir, l.name)
mfs.copy_from_local(r, path, target, hdfs_user)
uploaded_paths.append(target)
for lib in builtin_libs:
mfs.put_file_to_maprfs(r, lib['raw'], lib['name'], lib_dir,
hdfs_user)
uploaded_paths.append(lib_dir + '/' + lib['name'])
return uploaded_paths
def create_blueprint(cluster):
_prepare_ranger(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
host_groups = []
for ng in cluster.node_groups:
procs = p_common.get_ambari_proc_list(ng)
procs.extend(p_common.get_clients(cluster))
for instance in ng.instances:
hg = {
"name": instance.instance_name,
"configurations": configs.get_instance_params(instance),
"components": get_host_group_components(cluster, procs)
}
host_groups.append(hg)
bp = {
"Blueprints": {
"stack_name": "HDP",
"stack_version": cluster.hadoop_version,
},
"host_groups": host_groups,
"configurations": configs.get_cluster_params(cluster)
}
if kerberos.is_kerberos_security_enabled(cluster):
bp['configurations'].extend([
_serialize_mit_kdc_kerberos_env(cluster),
_serialize_krb5_configs(cluster)
])
bp['Blueprints']['security'] = {'type': 'KERBEROS'}
general_configs = cluster.cluster_configs.get("general", {})
if (general_configs.get(p_common.NAMENODE_HA)
or general_configs.get(p_common.RESOURCEMANAGER_HA)
or general_configs.get(p_common.HBASE_REGIONSERVER_HA)):
bp = ha_helper.update_bp_ha_common(cluster, bp)
if general_configs.get(p_common.NAMENODE_HA):
bp = ha_helper.update_bp_for_namenode_ha(cluster, bp)
if general_configs.get(p_common.RESOURCEMANAGER_HA):
bp = ha_helper.update_bp_for_resourcemanager_ha(cluster, bp)
if general_configs.get(p_common.HBASE_REGIONSERVER_HA):
bp = ha_helper.update_bp_for_hbase_ha(cluster, bp)
with _get_ambari_client(cluster) as client:
return client.create_blueprint(cluster.name, bp)
def test_validate_additional_ng_scaling_raises(self):
data = [{
'name': 'master',
'flavor_id': '42',
'count': 1,
'node_processes': ['nimbus']
}, {
'name': 'slave',
'flavor_id': '42',
'count': 1,
'node_processes': ['supervisor']
}, {
'name': 'zookeeper',
'flavor_id': '42',
'count': 1,
'node_processes': ['zookeeper']
}, {
'name': 'master2',
'flavor_id': '42',
'count': 0,
'node_processes': ['nimbus']
}]
cluster_data_101 = self._get_cluster('cluster_1.0.1', '1.0.1')
cluster_data_110 = self._get_cluster('cluster_1.1.0', '1.1.0')
cluster_data_120 = self._get_cluster('cluster_1.2.0', '1.2')
cluster_data_101['node_groups'] = data
cluster_data_110['node_groups'] = data
cluster_data_120['node_groups'] = data
clusters = [cluster_data_101, cluster_data_110, cluster_data_120]
for cluster_data in clusters:
cluster = conductor.cluster_create(context.ctx(), cluster_data)
plugin = pb.PLUGINS.get_plugin(cluster.plugin_name)
master_id = [
node.id for node in cluster.node_groups
if node.name == 'master2'
]
self.assertRaises(ex.NodeGroupCannotBeScaled,
plugin._validate_existing_ng_scaling, cluster,
master_id)
def _set_cluster_info(self, cluster):
nn = vu.get_namenode(cluster)
rm = vu.get_resourcemanager(cluster)
hs = vu.get_historyserver(cluster)
oo = vu.get_oozie(cluster)
sp = vu.get_spark_history_server(cluster)
info = {}
if rm:
info['YARN'] = {
'Web UI':
'http://%s:%s' % (rm.get_ip_or_dns_name(), '8088'),
'ResourceManager':
'http://%s:%s' % (rm.get_ip_or_dns_name(), '8032')
}
if nn:
info['HDFS'] = {
'Web UI': 'http://%s:%s' % (nn.get_ip_or_dns_name(), '50070'),
'NameNode': 'hdfs://%s:%s' % (nn.hostname(), '9000')
}
if oo:
info['JobFlow'] = {
'Oozie': 'http://%s:%s' % (oo.get_ip_or_dns_name(), '11000')
}
if hs:
info['MapReduce JobHistory Server'] = {
'Web UI': 'http://%s:%s' % (hs.get_ip_or_dns_name(), '19888')
}
if sp:
info['Apache Spark'] = {
'Spark UI':
'http://%s:%s' % (sp.management_ip, '4040'),
'Spark History Server UI':
'http://%s:%s' % (sp.management_ip, '18080')
}
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {'info': info})
def test_plugin_edp_engine_no_spark(self):
cluster_dict = {
'name': 'cluster',
'plugin_name': 'cdh',
'hadoop_version': '5.7.0',
'default_image_id': 'image'}
job = mock.Mock()
job.type = edp.JOB_TYPE_SPARK
cluster = conductor.cluster_create(context.ctx(), cluster_dict)
plugin = pb.PLUGINS.get_plugin(cluster.plugin_name)
edp_engine = plugin.get_edp_engine(cluster, edp.JOB_TYPE_SPARK)
with testtools.ExpectedException(
ex.InvalidComponentCountException,
value_re="Hadoop cluster should contain 1 "
"SPARK_YARN_HISTORY_SERVER component\(s\). Actual "
"SPARK_YARN_HISTORY_SERVER count is 0\nError ID: .*"):
edp_engine.validate_job_execution(cluster, job, mock.Mock())
def get_password_from_db(cluster, pwname):
"""return a password for the named entry
This function will return, or create and return, a password for the
named entry. It will store the password in the key manager and use
the ID in the database entry.
:param cluster: The cluster record containing the password
:param pwname: The entry name associated with the password
:returns: The cleartext password
"""
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster.id)
passwd = cluster.extra.get(pwname) if cluster.extra else None
if passwd:
return key_manager.get_secret(passwd, ctx)
passwd = uuidutils.generate_uuid()
extra = cluster.extra.to_dict() if cluster.extra else {}
extra[pwname] = key_manager.store_secret(passwd, ctx)
conductor.cluster_update(ctx, cluster, {'extra': extra})
return passwd
def provision_keypairs(cluster, instances=None):
extra = cluster.extra.to_dict() if cluster.extra else {}
# use same keypair for scaling
keypair = extra.get('vanilla_keypair')
if not instances:
instances = utils.get_instances(cluster)
else:
# scaling
if not keypair:
# cluster created before mitaka, skipping provisioning
return
if not keypair:
private, public = utils.generate_key_pair()
keypair = {'public': public, 'private': private}
extra['vanilla_keypair'] = keypair
extra['vanilla_keypair']['private'] = _store_secret(keypair['private'])
conductor.cluster_update(context.ctx(), cluster, {'extra': extra})
else:
keypair['private'] = _get_secret(keypair['private'])
with context.PluginsThreadGroup() as tg:
for instance in instances:
tg.spawn('provision-key-%s' % instance.instance_name,
_provision_key, instance, keypair)
def _test_engine(self, version, job_type, eng):
cluster_dict = self._get_cluster('demo', version)
cluster = conductor.cluster_create(context.ctx(), cluster_dict)
plugin = pb.PLUGINS.get_plugin(cluster.plugin_name)
self.assertIsInstance(plugin.get_edp_engine(cluster, job_type), eng)
def configure_cluster(self, cluster):
self.deploy.configure_cluster(cluster)
conductor.cluster_update(
context.ctx(), cluster, {
'info':
self.cloudera_utils.get_cloudera_manager_info(cluster)})
def _set_cluster_info(self, cluster):
ambari_ip = plugin_utils.get_instance(
cluster, p_common.AMBARI_SERVER).get_ip_or_dns_name()
ambari_port = "8080"
info = {
p_common.AMBARI_SERVER: {
"Web UI": "http://{host}:{port}".format(host=ambari_ip,
port=ambari_port),
"Username": "******",
"Password": cluster.extra["ambari_password"]
}
}
nns = plugin_utils.get_instances(cluster, p_common.NAMENODE)
info[p_common.NAMENODE] = {}
for idx, namenode in enumerate(nns):
info[p_common.NAMENODE][
"Web UI %s" % (idx + 1)] = (
"http://%s:50070" % namenode.get_ip_or_dns_name())
rms = plugin_utils.get_instances(cluster, p_common.RESOURCEMANAGER)
info[p_common.RESOURCEMANAGER] = {}
for idx, resourcemanager in enumerate(rms):
info[p_common.RESOURCEMANAGER][
"Web UI %s" % (idx + 1)] = (
"http://%s:8088" % resourcemanager.get_ip_or_dns_name())
historyserver = plugin_utils.get_instance(cluster,
p_common.HISTORYSERVER)
if historyserver:
info[p_common.HISTORYSERVER] = {
"Web UI": "http://%s:19888" %
historyserver.get_ip_or_dns_name()
}
atlserver = plugin_utils.get_instance(cluster,
p_common.APP_TIMELINE_SERVER)
if atlserver:
info[p_common.APP_TIMELINE_SERVER] = {
"Web UI": "http://%s:8188" % atlserver.get_ip_or_dns_name()
}
oozie = plugin_utils.get_instance(cluster, p_common.OOZIE_SERVER)
if oozie:
info[p_common.OOZIE_SERVER] = {
"Web UI": "http://%s:11000/oozie" % oozie.get_ip_or_dns_name()
}
hbase_master = plugin_utils.get_instance(cluster,
p_common.HBASE_MASTER)
if hbase_master:
info[p_common.HBASE_MASTER] = {
"Web UI": "http://%s:16010" % hbase_master.get_ip_or_dns_name()
}
falcon = plugin_utils.get_instance(cluster, p_common.FALCON_SERVER)
if falcon:
info[p_common.FALCON_SERVER] = {
"Web UI": "http://%s:15000" % falcon.get_ip_or_dns_name()
}
storm_ui = plugin_utils.get_instance(cluster, p_common.STORM_UI_SERVER)
if storm_ui:
info[p_common.STORM_UI_SERVER] = {
"Web UI": "http://%s:8744" % storm_ui.get_ip_or_dns_name()
}
ranger_admin = plugin_utils.get_instance(cluster,
p_common.RANGER_ADMIN)
if ranger_admin:
info[p_common.RANGER_ADMIN] = {
"Web UI": "http://%s:6080" % ranger_admin.get_ip_or_dns_name(),
"Username": "******",
"Password": "******"
}
spark_hs = plugin_utils.get_instance(cluster,
p_common.SPARK_JOBHISTORYSERVER)
if spark_hs:
info[p_common.SPARK_JOBHISTORYSERVER] = {
"Web UI": "http://%s:18080" % spark_hs.get_ip_or_dns_name()
}
info.update(cluster.info.to_dict())
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {"info": info})
cluster = conductor.cluster_get(ctx, cluster.id)
