def jdbc_connector():
import params
if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
environment = {"no_proxy": format("{ambari_server_hostname}")}
# TODO: should be removed after ranger_hive_plugin will not provide jdbc
Execute(('rm', '-f', params.prepackaged_ojdbc_symlink),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source))
# maybe it will be more correcvly to use db type
if params.sqla_db_used:
untar_sqla_type2_driver = ('tar', '-xvf',
params.downloaded_custom_connector,
'-C', params.tmp_dir)
Execute(untar_sqla_type2_driver, sudo=True)
Execute(as_sudo([
'yes', '|', 'cp', params.jars_path_in_archive, params.hive_lib
],
auto_escape=False),
path=["/bin", "/usr/bin/"])
Directory(params.jdbc_libs_dir, recursive=True)
Execute(as_sudo([
'yes', '|', 'cp', params.libs_path_in_archive,
params.jdbc_libs_dir
],
auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(
('cp', '--remove-destination',
params.downloaded_custom_connector, params.target),
#creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
path=["/bin", "/usr/bin/"],
sudo=True)
else:
#for default hive db (Mysql)
Execute(
('cp', '--remove-destination',
format('/usr/share/java/{jdbc_jar_name}'), params.target),
#creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
path=["/bin", "/usr/bin/"],
sudo=True)
File(
params.target,
mode=0644,
)
def enable_keep_alive(self):
httpd_conf_dir = self.httpd_conf_dir
command = format(
"grep -E 'KeepAlive (On|Off)' {httpd_conf_dir}/httpd.conf && " +
as_sudo(('sed', '-i', 's/KeepAlive Off/KeepAlive On/',
format("{httpd_conf_dir}/httpd.conf"))) +
" || echo 'KeepAlive On' | ") + as_sudo(
('tee', '--append',
format('{httpd_conf_dir}/httpd.conf'))) + " > /dev/null"
Execute(command)
def checkAndStopRegistyDNS(cmd):
import params
import status_params
hadoop_env_exports = {'HADOOP_LIBEXEC_DIR': params.hadoop_libexec_dir}
componentName = 'registrydns'
daemon_cmd = format("{cmd} stop {componentName}")
log_dir = params.yarn_log_dir
# When registry dns is switched from non-privileged to privileged mode or the other way,
# then the previous instance of registry dns has a different pid/user.
# Checking if either of the processes are running and shutting them down if they are.
for dns_pid_file, dns_user in [
(status_params.yarn_registry_dns_priv_pid_file,
status_params.root_user),
(status_params.yarn_registry_dns_pid_file, params.yarn_user)
]:
process_id_exists_command = as_sudo([
"test", "-f", dns_pid_file
]) + " && " + as_sudo(["pgrep", "-F", dns_pid_file])
try:
Execute(daemon_cmd,
only_if=process_id_exists_command,
user=dns_user)
except:
# When the registry dns port is modified but registry dns is not started
# immediately, then the configs in yarn-env.sh & yarn-site.xml related
# to registry dns may have already changed. This introduces a discrepancy
# between the actual process that is running and the configs.
# For example, when port is changed from 5300 to 53,
# then dns port = 53 in yarn-site and YARN_REGISTRYDNS_SECURE_* envs in yarn-env.sh
# are saved. So, while trying to shutdown the stray non-privileged registry dns process
# after sometime, yarn daemon from the configs thinks that it needs privileged
# access and throws an exception. In such cases, we try to kill the stray process.
show_logs(log_dir, dns_user)
pass
process_id_does_not_exist_command = format(
"! ( {process_id_exists_command} )")
code, out = shell.call(process_id_does_not_exist_command,
env=hadoop_env_exports,
tries=5,
try_sleep=5)
if code != 0:
code, out, err = shell.checked_call(("cat", dns_pid_file),
sudo=True,
env=hadoop_env_exports,
stderr=subprocess32.PIPE)
Logger.info("PID to kill was retrieved: '" + out + "'.")
out = out.splitlines()[0]
pid = out
Execute(("kill", "-9", pid), sudo=True)
File(dns_pid_file, action="delete")
def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
pid_file = getPid(params, self.nodeType)
process_id_exists_command = as_sudo(["test", "-f", pid_file]) + " && " + as_sudo(["pgrep", "-F", pid_file])
daemon_cmd = get_daemon_cmd(params, self.nodeType, "stop")
try:
Execute(daemon_cmd,
user=params.druid_user,
only_if=process_id_exists_command,
)
except:
show_logs(params.druid_log_dir, params.druid_user)
raise
def copy_jdbc_connector(kms_home):
import params
if params.jdbc_jar_name is None and params.driver_curl_source.endswith("/None"):
error_message = "Error! Sorry, but we can't find jdbc driver related to {0} database to download from {1}. \
Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'".format(params.db_flavor, params.jdk_location)
Logger.error(error_message)
if params.driver_curl_source and not params.driver_curl_source.endswith("/None"):
if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar):
File(params.previous_jdbc_jar, action='delete')
driver_curl_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
mode = 0644
)
Directory(os.path.join(kms_home, 'ews', 'lib'),
mode=0755
)
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
Directory(params.jdbc_libs_dir,
cd_access="a",
create_parents=True)
Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
path=["/bin", "/usr/bin/"])
File(os.path.join(kms_home, 'ews', 'webapp', 'lib', 'sajdbc4.jar'), mode=0644)
else:
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(kms_home, 'ews', 'webapp', 'lib', params.jdbc_jar_name), mode=0644)
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = params.config['configurations']['kms-properties'],
owner = params.kms_user
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{kms_home}/ews/webapp/lib/sajdbc4.jar')},
owner = params.kms_user,
)
else:
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
owner = params.kms_user,
)
def link_config(old_conf, link_conf):
"""
Creates a config link following:
1. Checks if the old_conf location exists
2. If it does, check if it's a link already
3. Make a copy to /etc/[component]/conf.backup
4. Remove the old directory and create a symlink to link_conf
:old_conf: the old config directory, ie /etc/[component]/config
:link_conf: the new target for the config directory, ie /usr/hdp/current/[component-dir]/conf
"""
if not os.path.exists(old_conf):
Logger.debug("Skipping {0}; it does not exist".format(old_conf))
return
if os.path.islink(old_conf):
Logger.debug("Skipping {0}; it is already a link".format(old_conf))
return
old_parent = os.path.abspath(os.path.join(old_conf, os.pardir))
Logger.info("Linking {0} to {1}".format(old_conf, link_conf))
old_conf_copy = os.path.join(old_parent, "conf.backup")
if not os.path.exists(old_conf_copy):
Execute(as_sudo(["cp", "-R", "-p", old_conf, old_conf_copy]),
logoutput=True)
shutil.rmtree(old_conf, ignore_errors=True)
# link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf
Link(old_conf, to=link_conf)
def setup_usersync(upgrade_type=None):
import params
PropertiesFile(format("{usersync_home}/install.properties"),
properties = params.config['configurations']['usersync-properties'],
)
custom_config = dict()
custom_config['unix_user'] = params.unix_user
custom_config['unix_group'] = params.unix_group
ModifyPropertiesFile(format("{usersync_home}/install.properties"),
properties=custom_config
)
cmd = format("cd {usersync_home} && ") + as_sudo([format('{usersync_home}/setup.sh')])
Execute(cmd, environment={'JAVA_HOME': params.java_home}, logoutput=True)
File([params.usersync_start, params.usersync_stop],
owner = params.unix_user
)
File(params.usersync_services_file,
mode = 0755,
)
Directory(params.usersync_log_dir,
owner = params.unix_user,
group = params.unix_group
)
def copy_jdbc_connector(stack_version=None):
import params
if params.jdbc_jar_name is None and params.driver_curl_source.endswith("/None"):
error_message = format("{db_flavor} jdbc driver cannot be downloaded from {jdk_location}\nPlease run 'ambari-server setup --jdbc-db={db_flavor} --jdbc-driver={{path_to_jdbc}}' on ambari-server host.")
raise Fail(error_message)
if params.driver_curl_source and not params.driver_curl_source.endswith("/None"):
if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar):
File(params.previous_jdbc_jar, action='delete')
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
mode = 0644
)
ranger_home = params.ranger_home
if stack_version is not None:
ranger_home = format("{stack_root}/{stack_version}/ranger-admin")
driver_curl_target = format("{ranger_home}/ews/lib/{jdbc_jar_name}")
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(ranger_home, 'ews', 'lib', 'sajdbc4.jar'), mode=0644)
Directory(params.jdbc_libs_dir,
cd_access="a",
create_parents=True)
Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(ranger_home, 'ews', 'lib',params.jdbc_jar_name), mode=0644)
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = params.config['configurations']['admin-properties'],
owner = params.unix_user,
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{ranger_home}/ews/lib/sajdbc4.jar')},
owner = params.unix_user,
)
else:
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
owner = params.unix_user,
)
def hbase_service(name, action='start'): # 'start' or 'stop' or 'status'
import params
role = name
cmd = format("{daemon_script} --config {hbase_conf_dir}")
pid_file = format("{pid_dir}/hbase-{hbase_user}-{role}.pid")
pid_expression = as_sudo(["cat", pid_file])
no_op_test = as_sudo([
"test", "-f", pid_file
]) + format(" && ps -p `{pid_expression}` >/dev/null 2>&1")
if action == 'start':
# Check ambari-metrics-hadoop-sink version is less than 2.7.0.0
check_installed_metrics_hadoop_sink_version()
daemon_cmd = format("{cmd} start {role}")
try:
Execute(daemon_cmd, not_if=no_op_test, user=params.hbase_user)
except:
show_logs(params.log_dir, params.hbase_user)
raise
elif action == 'stop':
daemon_cmd = format("{cmd} stop {role}")
try:
Execute(
daemon_cmd,
user=params.hbase_user,
only_if=no_op_test,
# BUGFIX: hbase regionserver sometimes hangs when nn is in safemode
timeout=params.hbase_regionserver_shutdown_timeout,
on_timeout=format(
"! ( {no_op_test} ) || {sudo} -H -E kill -9 `{pid_expression}`"
),
)
except:
show_logs(params.log_dir, params.hbase_user)
raise
File(
pid_file,
action="delete",
)
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
install_druid()
self.configure(env, upgrade_type=upgrade_type)
daemon_cmd = get_daemon_cmd(params, self.nodeType, "start")
# Verify Database connection on Druid start
if params.metadata_storage_type == 'mysql':
if not params.jdbc_driver_jar or not os.path.isfile(
params.connector_download_dir + os.path.sep +
params.jdbc_driver_jar):
path_to_jdbc = params.connector_download_dir + os.path.sep + "*"
error_message = "Error! Sorry, but we can't find jdbc driver for mysql.So, db connection check can fail." + \
"Please run 'ambari-server setup --jdbc-db=mysql --jdbc-driver={path_to_jdbc} on server host.'"
Logger.error(error_message)
else:
path_to_jdbc = params.connector_download_dir + os.path.sep + params.jdbc_driver_jar
db_connection_check_command = format(
"{params.java8_home}/bin/java -cp {params.check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{params.metadata_storage_url}' {params.metadata_storage_user} {params.metadata_storage_password!p} com.mysql.jdbc.Driver"
)
else:
db_connection_check_command = None
if db_connection_check_command:
sudo.chmod(params.check_db_connection_jar, 0755)
Execute(
db_connection_check_command,
tries=5,
try_sleep=10,
user=params.druid_user)
pid_file = getPid(params, self.nodeType)
process_id_exists_command = as_sudo([
"test", "-f", pid_file
]) + " && " + as_sudo(["pgrep", "-F", pid_file])
try:
Execute(
daemon_cmd,
user=params.druid_user,
not_if=process_id_exists_command,
)
except:
show_logs(params.druid_log_dir, params.druid_user)
raise
def hbase_service(name, action='start'): # 'start' or 'stop'
import params
sudo = AMBARI_SUDO_BINARY
daemon_script = format("{yarn_hbase_bin}/hbase-daemon.sh")
role = name
cmd = format("{daemon_script} --config {yarn_hbase_conf_dir}")
pid_file = format(
"{yarn_hbase_pid_dir}/hbase-{yarn_hbase_user}-{role}.pid")
pid_expression = as_sudo(["cat", pid_file])
no_op_test = as_sudo([
"test", "-f", pid_file
]) + format(" && ps -p `{pid_expression}` >/dev/null 2>&1")
if action == 'start':
daemon_cmd = format("{cmd} start {role}")
try:
Execute(daemon_cmd, not_if=no_op_test, user=params.yarn_hbase_user)
except:
show_logs(params.yarn_hbase_log_dir, params.yarn_hbase_user)
raise
elif action == 'stop':
daemon_cmd = format("{cmd} stop {role}")
try:
Execute(
daemon_cmd,
user=params.yarn_hbase_user,
only_if=no_op_test,
timeout=30,
on_timeout=format(
"! ( {no_op_test} ) || {sudo} -H -E kill -9 `{pid_expression}`"
),
)
except:
show_logs(params.yarn_hbase_log_dir, params.yarn_hbase_user)
raise
File(
pid_file,
action="delete",
)
def setup_hdp_install_directory():
# This is a name of marker file.
SELECT_ALL_PERFORMED_MARKER = "/var/lib/ambari-agent/data/hdp-select-set-all.performed"
import params
if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0:
Execute(as_sudo(['touch', SELECT_ALL_PERFORMED_MARKER]) + ' ; ' +
format('{sudo} /usr/bin/hdp-select set all `ambari-python-wrap /usr/bin/hdp-select versions | grep ^{stack_version_unformatted} | tail -1`'),
only_if=format('ls -d /usr/hdp/{stack_version_unformatted}*'), # If any HDP version is installed
not_if=format("test -f {SELECT_ALL_PERFORMED_MARKER}") # Do that only once (otherwise we break rolling upgrade logic)
)
def setup_hdp_install_directory():
# This is a name of marker file.
SELECT_ALL_PERFORMED_MARKER = "/var/lib/ambari-agent/data/hdp-select-set-all.performed"
import params
if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0:
Execute(as_sudo(['touch', SELECT_ALL_PERFORMED_MARKER]) + ' ; ' +
format('{sudo} /usr/bin/hdp-select set all `ambari-python-wrap /usr/bin/hdp-select versions | grep ^{stack_version_unformatted} | tail -1`'),
only_if=format('ls -d /usr/hdp/{stack_version_unformatted}*'), # If any HDP version is installed
not_if=format("test -f {SELECT_ALL_PERFORMED_MARKER}") # Do that only once (otherwise we break rolling upgrade logic)
)
def stop(self, env):
import params
env.set_params(params)
self.configure(env)
daemon_cmd = self.get_daemon_cmd(params, "stop")
pid_file = params.superset_pid_dir + '/superset.pid'
process_id_exists_command = as_sudo([
"test", "-f", pid_file
]) + " && " + as_sudo(["pgrep", "-F", pid_file])
try:
Execute(
daemon_cmd,
user=params.superset_user,
only_if=process_id_exists_command,
)
except:
show_logs(params.superset_log_dir, params.superset_user)
raise
def hbase_service(name, action='start'):
import params
role = name
cmd = format("{daemon_script} --config {hbase_conf_dir}")
pid_file = format("{pid_dir}/hbase-{hbase_user}-{role}.pid")
pid_expression = as_sudo(["cat", pid_file])
no_op_test = as_sudo([
"test", "-f", pid_file
]) + format(" && ps -p `{pid_expression}` >/dev/null 2>&1")
# delete wal log if HBase version has moved down
if params.to_backup_wal_dir:
wal_directory = params.wal_directory
timestamp = datetime.now()
timestamp_format = '%Y%m%d%H%M%S'
wal_directory_backup = '%s_%s' % (wal_directory,
timestamp.strftime(timestamp_format))
check_if_wal_dir_exists = format("hdfs dfs -ls {wal_directory}")
wal_dir_exists = False
try:
Execute(check_if_wal_dir_exists, user=params.hbase_user)
wal_dir_exists = True
except Exception, e:
Logger.error(
format(
"Did not find HBase WAL directory {wal_directory}. It's possible that it was already moved. Exception: {e.message}"
))
if wal_dir_exists:
move_wal_dir_cmd = format(
"hdfs dfs -mv {wal_directory} {wal_directory_backup}")
try:
Execute(move_wal_dir_cmd, user=params.hbase_user)
except Exception, e:
Logger.error(
format(
"Failed to backup HBase WAL directory, command: {move_wal_dir_cmd} . Exception: {e.message}"
))
def copy_jdbc_connector(stack_version=None):
import params
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
mode = 0644
)
ranger_home = params.ranger_home
if stack_version is not None:
ranger_home = format("{stack_root}/{stack_version}/ranger-admin")
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(ranger_home, 'ews', 'lib', 'sajdbc4.jar'), mode=0644)
Directory(params.jdbc_libs_dir,
cd_access="a",
create_parents=True)
Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(ranger_home, 'ews', 'lib',params.jdbc_jar_name), mode=0644)
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = params.config['configurations']['admin-properties'],
owner = params.unix_user,
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{ranger_home}/ews/lib/sajdbc4.jar')},
owner = params.unix_user,
)
else:
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
owner = params.unix_user,
)
def setup_ranger_admin(upgrade_type=None):
import params
check_db_connnection()
File(params.driver_curl_target, mode=0644)
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties=params.config['configurations']['admin-properties']
)
custom_config = dict()
custom_config['unix_user'] = params.unix_user
custom_config['unix_group'] = params.unix_group
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties=custom_config
)
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties={'SQL_CONNECTOR_JAR': format('{driver_curl_target}')}
)
##if db flavor == oracle - set oracle home env variable
if params.db_flavor.lower() == 'oracle' and params.oracle_home:
env_dict = {'JAVA_HOME': params.java_home, 'ORACLE_HOME': params.oracle_home,
'LD_LIBRARY_PATH': params.oracle_home}
else:
env_dict = {'JAVA_HOME': params.java_home}
setup_sh = format("cd {ranger_home} && ") + as_sudo([format('{ranger_home}/setup.sh')])
Execute(setup_sh,
environment=env_dict,
logoutput=True,
)
ModifyPropertiesFile(format("{ranger_conf}/xa_system.properties"),
properties=params.config['configurations']['ranger-site'],
)
ModifyPropertiesFile(format("{ranger_conf}/ranger_webserver.properties"),
properties=params.config['configurations']['ranger-site'],
mode=0744
)
Directory(params.admin_log_dir,
owner=params.unix_user,
group=params.unix_group
)
def _copy_configurations(source_directory, target_directory):
"""
Copies from the source directory to the target directory. If the source directory is a symlink
then it will be followed (deferenced) but any other symlinks found to copy will not be. This
will ensure that if the configuration directory itself is a symlink, then it's contents will be
copied, preserving and children found which are also symlinks.
:param source_directory: the source directory to copy from
:param target_directory: the target directory to copy to
:return: None
"""
# append trailing slash so the cp command works correctly WRT recursion and symlinks
source_directory = os.path.join(source_directory, "*")
Execute(as_sudo(["cp", "-R", "-p", "-v", source_directory, target_directory], auto_escape = False),
logoutput = True)
def create_30_config_version(self, env):
package_name = 'registry'
stack_root = Script.get_stack_root()
current_dir = "{0}/current/registry/conf".format(stack_root)
directories = [{
"conf_dir": "/etc/registry/conf",
"current_dir": current_dir
}]
stack_version = stack_select.get_stack_version_before_install(
package_name)
conf_dir = "/etc/registry/conf"
if stack_version:
try:
#Check if broken symbolic links issue exists
os.stat(conf_dir)
conf_select.convert_conf_directories_to_symlinks(
package_name, stack_version, directories)
cp_cmd = as_sudo([
"cp", "-a", "-f", "/etc/registry/conf.backup/.",
"/etc/registry/conf"
])
Execute(cp_cmd, logoutput=True)
except OSError as e:
Logger.warning(
"Detected broken symlink : {0}. Attempting to repair.".
format(str(e)))
#removing symlink conf directory
sudo.unlink(conf_dir)
#make conf dir again
sudo.makedirs(conf_dir, 0755)
#copy all files
for files in glob.glob("/etc/registry/conf.backup/*"):
cp_cmd = as_sudo(["cp", "-r", files, conf_dir])
Execute(cp_cmd, logoutput=True)
conf_select.convert_conf_directories_to_symlinks(
package_name, stack_version, directories)
def _copy_configurations(source_directory, target_directory):
"""
Copies from the source directory to the target directory. If the source directory is a symlink
then it will be followed (deferenced) but any other symlinks found to copy will not be. This
will ensure that if the configuration directory itself is a symlink, then it's contents will be
copied, preserving and children found which are also symlinks.
:param source_directory: the source directory to copy from
:param target_directory: the target directory to copy to
:return: None
"""
# append trailing slash so the cp command works correctly WRT recursion and symlinks
source_directory = os.path.join(source_directory, "*")
Execute(as_sudo(["cp", "-R", "-p", "-v", source_directory, target_directory], auto_escape = False),
logoutput = True)
def _link_configs(package, version, old_conf, link_conf):
"""
Link a specific package's configuration directory
"""
if not os.path.exists(old_conf):
Logger.debug("Skipping {0} as it does not exist.".format(old_conf))
return
# check if conf is a link to the target already
if os.path.islink(old_conf):
Logger.debug("{0} is already a link to {1}".format(old_conf, os.path.realpath(old_conf)))
return
# make backup dir and copy everything in case configure() was called after install()
old_parent = os.path.abspath(os.path.join(old_conf, os.pardir))
old_conf_copy = os.path.join(old_parent, "conf.install")
Execute(("cp", "-R", "-p", old_conf, old_conf_copy),
not_if = format("test -e {old_conf_copy}"),
sudo = True,
)
versioned_conf = conf_select.create("HDP", package, version, dry_run = True)
Logger.info("New conf directory is {0}".format(versioned_conf))
# make new conf dir and copy everything in case configure() was called after install()
if not os.path.exists(versioned_conf):
conf_select.create("HDP", package, version)
Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False),
only_if = format("ls {old_conf}/*")
)
# make /usr/hdp/<version>/hadoop/conf point to the versioned config.
# /usr/hdp/current is already set
conf_select.select("HDP", package, version)
# no more references to /etc/[component]/conf
Directory(old_conf,
action="delete",
)
# link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf
Link(old_conf,
to = link_conf
)
def link_config(old_conf, link_conf):
"""
Creates a config link following:
1. Checks if the old_conf location exists
2. If it does, check if it's a link already
3. Make a copy to /etc/[component]/conf.backup
4. Remove the old directory and create a symlink to link_conf
:old_conf: the old config directory, ie /etc/[component]/conf
:link_conf: the new target for the config directory, ie /usr/hdp/current/[component-dir]/conf
"""
# if the link exists but is wrong, then change it
if os.path.islink(old_conf) and os.path.realpath(old_conf) != link_conf:
Link(old_conf, to = link_conf)
return
if os.path.islink(old_conf):
Logger.debug("Skipping {0}; it is already a link".format(old_conf))
return
if not os.path.exists(old_conf):
Logger.debug("Skipping {0}; it does not exist".format(old_conf))
return
old_parent = os.path.abspath(os.path.join(old_conf, os.pardir))
Logger.info("Linking {0} to {1}".format(old_conf, link_conf))
old_conf_copy = os.path.join(old_parent, "conf.backup")
if not os.path.exists(old_conf_copy):
Execute(as_sudo(["cp", "-R", "-p", old_conf, old_conf_copy]), logoutput=True)
shutil.rmtree(old_conf, ignore_errors=True)
# link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf
Link(old_conf, to = link_conf)
def enable_keep_alive(self):
httpd_conf_dir = self.httpd_conf_dir
command = format("grep -E 'KeepAlive (On|Off)' {httpd_conf_dir}/httpd.conf && " + as_sudo(('sed', '-i','s/KeepAlive Off/KeepAlive On/', format("{httpd_conf_dir}/httpd.conf"))) + " || echo 'KeepAlive On' | ") + as_sudo(('tee', '--append', format('{httpd_conf_dir}/httpd.conf'))) + " > /dev/null"
Execute(command
)
def spark_service(name, upgrade_type=None, action=None):
import params
if action == 'start':
check_sac_jar()
effective_version = params.version if upgrade_type is not None else params.stack_version_formatted
if effective_version:
effective_version = format_stack_version(effective_version)
if name == 'jobhistoryserver' and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version):
# create & copy spark2-hdp-yarn-archive.tar.gz to hdfs
if not params.sysprep_skip_copy_tarballs_hdfs:
source_dirs = [params.spark_home + "/jars"]
# include sac jar and spark-job keytab to archive
if params.sac_enabled:
if params.security_enabled:
shutil.copy(params.atlas_kafka_keytab, source_dirs[0])
os.chmod(os.path.join(source_dirs[0], os.path.basename(params.atlas_kafka_keytab)), 0440)
source_dirs.append(params.spark_atlas_jar_dir)
tmp_archive_file=get_tarball_paths("spark2")[1]
make_tarfile(tmp_archive_file, source_dirs)
copy_to_hdfs("spark2", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs, replace_existing_files=True)
if params.sac_enabled and params.security_enabled:
os.remove(os.path.join(source_dirs[0], os.path.basename(params.atlas_kafka_keytab)))
# create & copy spark2-hdp-hive-archive.tar.gz to hdfs
if not params.sysprep_skip_copy_tarballs_hdfs:
source_dirs = [params.spark_home+"/standalone-metastore"]
tmp_archive_file=get_tarball_paths("spark2hive")[1]
make_tarfile(tmp_archive_file, source_dirs)
copy_to_hdfs("spark2hive", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs, replace_existing_files=True)
# create spark history directory
params.HdfsResource(params.spark_history_dir,
type="directory",
action="create_on_execute",
owner=params.spark_user,
group=params.user_group,
mode=0777,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
if params.security_enabled:
spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
Execute(spark_kinit_cmd, user=params.spark_user)
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
if params.stack_version_formatted and check_stack_feature(StackFeature.TEZ_FOR_SPARK, params.stack_version_formatted):
resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs)
if resource_created:
params.HdfsResource(None, action="execute")
if name == 'jobhistoryserver':
create_catalog_cmd = format("{hive_schematool_bin}/schematool -dbType {hive_metastore_db_type} "
"-createCatalog {default_metastore_catalog} "
"-catalogDescription 'Default catalog, for Spark' -ifNotExists "
"-catalogLocation {default_fs}{spark_warehouse_dir}")
Execute(create_catalog_cmd,
user = params.hive_user)
historyserver_no_op_test = as_sudo(["test", "-f", params.spark_history_server_pid_file]) + " && " + as_sudo(["pgrep", "-F", params.spark_history_server_pid_file])
try:
Execute(params.spark_history_server_start,
user=params.spark_user,
environment={'JAVA_HOME': params.java_home},
not_if=historyserver_no_op_test)
except:
show_logs(params.spark_log_dir, user=params.spark_user)
raise
elif name == 'sparkthriftserver':
import status_params
if params.security_enabled:
hive_kinit_cmd = format("{kinit_path_local} -kt {hive_kerberos_keytab} {hive_kerberos_principal}; ")
Execute(hive_kinit_cmd, user=params.spark_user)
thriftserver_no_op_test= as_sudo(["test", "-f", params.spark_thrift_server_pid_file]) + " && " + as_sudo(["pgrep", "-F", params.spark_thrift_server_pid_file])
try:
Execute(format('{spark_thrift_server_start} --properties-file {spark_thrift_server_conf_file} {spark_thrift_cmd_opts_properties}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home},
not_if=thriftserver_no_op_test
)
except:
show_logs(params.spark_log_dir, user=params.spark_user)
raise
hive_connection_created = False
i = 0
while i < 15:
time.sleep(30)
Logger.info("Check connection to STS is created.")
beeline_url = ["jdbc:hive2://{fqdn}:{spark_thrift_port}/default"]
if params.security_enabled:
beeline_url.append("principal={hive_kerberos_principal}")
beeline_url.append("transportMode={spark_transport_mode}")
if params.spark_transport_mode.lower() == 'http':
beeline_url.append("httpPath={spark_thrift_endpoint}")
if params.spark_thrift_ssl_enabled:
beeline_url.append("ssl=true")
beeline_cmd = os.path.join(params.spark_home, "bin", "beeline")
cmd = "! %s -u '%s' -e '' 2>&1| awk '{print}'|grep -i -e 'Connection refused' -e 'Invalid URL' -e 'Error: Could not open'" % \
(beeline_cmd, format(";".join(beeline_url)))
try:
Execute(cmd, user=params.spark_user, path=[beeline_cmd], timeout=CHECK_COMMAND_TIMEOUT_DEFAULT)
hive_connection_created = True
Logger.info("Connection to STS is created.")
break
except:
Logger.info("Connection to STS still is not created.")
pass
Logger.info("Check STS process status.")
check_process_status(status_params.spark_thrift_server_pid_file)
i+=1
if not hive_connection_created:
raise ComponentIsNotRunning("Something goes wrong, STS connection was not created but STS process still alive. "
"Potential problems: Hive/YARN doesn't work correctly or too slow. For more information check STS logs.")
elif action == 'stop':
if name == 'jobhistoryserver':
try:
Execute(format('{spark_history_server_stop}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home}
)
except:
show_logs(params.spark_log_dir, user=params.spark_user)
raise
File(params.spark_history_server_pid_file,
action="delete"
)
elif name == 'sparkthriftserver':
try:
Execute(format('{spark_thrift_server_stop}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home}
)
except:
show_logs(params.spark_log_dir, user=params.spark_user)
raise
File(params.spark_thrift_server_pid_file,
action="delete"
)
def service(action=None,
name=None,
user=None,
options="",
create_pid_dir=False,
create_log_dir=False):
"""
:param action: Either "start" or "stop"
:param name: Component name, e.g., "namenode", "datanode", "secondarynamenode", "zkfc"
:param user: User to run the command as
:param options: Additional options to pass to command as a string
:param create_pid_dir: Create PID directory
:param create_log_dir: Crate log file directory
"""
import params
options = options if options else ""
pid_dir = format("{hadoop_pid_dir_prefix}/{user}")
pid_file = format("{pid_dir}/hadoop-{user}-{name}.pid")
hadoop_env_exports = {'HADOOP_LIBEXEC_DIR': params.hadoop_libexec_dir}
log_dir = format("{hdfs_log_dir_prefix}/{user}")
# NFS GATEWAY is always started by root using jsvc due to rpcbind bugs
# on Linux such as CentOS6.2. https://bugzilla.redhat.com/show_bug.cgi?id=731542
if name == "nfs3":
pid_file = format("{pid_dir}/hadoop_privileged_nfs3.pid")
custom_export = {
'HADOOP_PRIVILEGED_NFS_USER': params.hdfs_user,
'HADOOP_PRIVILEGED_NFS_PID_DIR': pid_dir,
'HADOOP_PRIVILEGED_NFS_LOG_DIR': log_dir
}
hadoop_env_exports.update(custom_export)
process_id_exists_command = as_sudo(
["test", "-f", pid_file]) + " && " + as_sudo(["pgrep", "-F", pid_file])
# on STOP directories shouldn't be created
# since during stop still old dirs are used (which were created during previous start)
if action != "stop":
if name == "nfs3":
Directory(params.hadoop_pid_dir_prefix,
mode=0755,
owner=params.root_user,
group=params.root_group)
else:
Directory(params.hadoop_pid_dir_prefix,
mode=0755,
owner=params.hdfs_user,
group=params.user_group)
if create_pid_dir:
Directory(pid_dir,
owner=user,
group=params.user_group,
create_parents=True)
if create_log_dir:
if name == "nfs3":
Directory(log_dir,
mode=0775,
owner=params.root_user,
group=params.user_group)
else:
Directory(log_dir,
owner=user,
group=params.user_group,
create_parents=True)
if params.security_enabled and name == "datanode":
## The directory where pid files are stored in the secure data environment.
hadoop_secure_dn_pid_dir = format(
"{hadoop_pid_dir_prefix}/{hdfs_user}")
hadoop_secure_dn_pid_file = format(
"{hadoop_secure_dn_pid_dir}/hadoop_secure_dn.pid")
# At datanode_non_root stack version and further, we may start datanode as a non-root even in secure cluster
if not (params.stack_version_formatted and check_stack_feature(
StackFeature.DATANODE_NON_ROOT, params.stack_version_formatted)
) or params.secure_dn_ports_are_in_use:
user = "******"
pid_file = format(
"{hadoop_pid_dir_prefix}/{hdfs_user}/hadoop-{hdfs_user}-{name}.pid"
)
if action == 'stop' and (params.stack_version_formatted and check_stack_feature(StackFeature.DATANODE_NON_ROOT, params.stack_version_formatted)) and \
os.path.isfile(hadoop_secure_dn_pid_file):
# We need special handling for this case to handle the situation
# when we configure non-root secure DN and then restart it
# to handle new configs. Otherwise we will not be able to stop
# a running instance
user = "******"
try:
check_process_status(hadoop_secure_dn_pid_file)
custom_export = {'HADOOP_SECURE_DN_USER': params.hdfs_user}
hadoop_env_exports.update(custom_export)
except ComponentIsNotRunning:
pass
hdfs_bin = format("{hadoop_bin}/hdfs")
if user == "root":
cmd = [
hdfs_bin, "--config", params.hadoop_conf_dir, "--daemon", action,
name
]
if options:
cmd += [
options,
]
daemon_cmd = as_sudo(cmd)
else:
cmd = format(
"{ulimit_cmd} {hdfs_bin} --config {hadoop_conf_dir} --daemon {action} {name}"
)
if options:
cmd += " " + options
daemon_cmd = as_user(cmd, user)
if action == "start":
# remove pid file from dead process
File(pid_file, action="delete", not_if=process_id_exists_command)
try:
Execute(daemon_cmd,
not_if=process_id_exists_command,
environment=hadoop_env_exports)
except:
show_logs(log_dir, user)
raise
elif action == "stop":
try:
Execute(daemon_cmd,
only_if=process_id_exists_command,
environment=hadoop_env_exports)
except:
show_logs(log_dir, user)
raise
wait_process_stopped(pid_file)
File(pid_file, action="delete")
def service(componentName, action='start', serviceName='yarn'):
import params
import status_params
if serviceName == 'mapreduce' and componentName == 'historyserver':
delete_pid_file = True
daemon = format("{mapred_bin}/mapred")
pid_file = format(
"{mapred_pid_dir}/hadoop-{mapred_user}-{componentName}.pid")
usr = params.mapred_user
log_dir = params.mapred_log_dir
cmd = format(
"export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --daemon"
)
else:
# !!! yarn-daemon.sh deletes the PID for us; if we remove it the script
# may not work correctly when stopping the service
delete_pid_file = True
daemon = format("{yarn_bin}/yarn")
if componentName == 'registrydns' and status_params.registry_dns_needs_privileged_access:
pid_file = status_params.yarn_registry_dns_priv_pid_file
usr = status_params.root_user
else:
pid_file = format(
"{yarn_pid_dir}/hadoop-{yarn_user}-{componentName}.pid")
usr = params.yarn_user
log_dir = params.yarn_log_dir
cmd = format(
"export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {hadoop_conf_dir} --daemon"
)
check_process = as_sudo(["test", "-f", pid_file]) + " && " + as_sudo(
["pgrep", "-F", pid_file])
if action == 'start':
daemon_cmd = format("{ulimit_cmd} {cmd} start {componentName}")
if componentName == 'registrydns':
checkAndStopRegistyDNS(cmd)
else:
# Remove the pid file if its corresponding process is not running.
File(pid_file, action="delete", not_if=check_process)
if componentName == 'timelineserver' and serviceName == 'yarn':
File(params.ats_leveldb_lock_file,
action="delete",
only_if=format("ls {params.ats_leveldb_lock_file}"),
not_if=check_process,
ignore_failures=True)
try:
# Attempt to start the process. Internally, this is skipped if the process is already running.
Execute(daemon_cmd, user=usr, not_if=check_process)
# Ensure that the process with the expected PID exists.
Execute(
check_process,
not_if=check_process,
tries=5,
try_sleep=1,
)
except:
show_logs(log_dir, usr)
raise
elif action == 'stop':
daemon_cmd = format("{cmd} stop {componentName}")
if componentName == 'registrydns':
checkAndStopRegistyDNS(cmd)
else:
try:
Execute(daemon_cmd, user=usr, only_if=check_process)
except:
show_logs(log_dir, usr)
raise
# !!! yarn-daemon doesn't need us to delete PIDs
if delete_pid_file is True:
File(pid_file, action="delete")
elif action == 'refreshQueues':
rm_kinit_cmd = params.rm_kinit_cmd
refresh_cmd = format(
"{rm_kinit_cmd} export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {yarn_bin}/yarn rmadmin -refreshQueues"
)
Execute(
refresh_cmd,
user=usr,
timeout=
20, # when Yarn is not started command hangs forever and should be killed
tries=5,
try_sleep=5,
timeout_kill_strategy=TerminateStrategy.
KILL_PROCESS_GROUP, # the process cannot be simply killed by 'kill -15', so kill pg group instread.
)
def service(action=None, name=None, user=None, options="", create_pid_dir=False,
create_log_dir=False):
"""
:param action: Either "start" or "stop"
:param name: Component name, e.g., "namenode", "datanode", "secondarynamenode", "zkfc"
:param user: User to run the command as
:param options: Additional options to pass to command as a string
:param create_pid_dir: Create PID directory
:param create_log_dir: Crate log file directory
"""
import params
options = options if options else ""
pid_dir = format("{hadoop_pid_dir_prefix}/{user}")
pid_file = format("{pid_dir}/hadoop-{user}-{name}.pid")
hadoop_env_exports = {
'HADOOP_LIBEXEC_DIR': params.hadoop_libexec_dir
}
log_dir = format("{hdfs_log_dir_prefix}/{user}")
# NFS GATEWAY is always started by root using jsvc due to rpcbind bugs
# on Linux such as CentOS6.2. https://bugzilla.redhat.com/show_bug.cgi?id=731542
if name == "nfs3" :
pid_file = format("{pid_dir}/hadoop_privileged_nfs3.pid")
custom_export = {
'HADOOP_PRIVILEGED_NFS_USER': params.hdfs_user,
'HADOOP_PRIVILEGED_NFS_PID_DIR': pid_dir,
'HADOOP_PRIVILEGED_NFS_LOG_DIR': log_dir
}
hadoop_env_exports.update(custom_export)
process_id_exists_command = as_sudo(["test", "-f", pid_file]) + " && " + as_sudo(["pgrep", "-F", pid_file])
# on STOP directories shouldn't be created
# since during stop still old dirs are used (which were created during previous start)
if action != "stop":
if name == "nfs3":
Directory(params.hadoop_pid_dir_prefix,
mode=0755,
owner=params.root_user,
group=params.root_group
)
else:
Directory(params.hadoop_pid_dir_prefix,
mode=0755,
owner=params.hdfs_user,
group=params.user_group
)
if create_pid_dir:
Directory(pid_dir,
owner=user,
recursive=True)
if create_log_dir:
if name == "nfs3":
Directory(log_dir,
mode=0775,
owner=params.root_user,
group=params.user_group)
else:
Directory(log_dir,
owner=user,
recursive=True)
if params.security_enabled and name == "datanode":
## The directory where pid files are stored in the secure data environment.
hadoop_secure_dn_pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}")
hadoop_secure_dn_pid_file = format("{hadoop_secure_dn_pid_dir}/hadoop_secure_dn.pid")
# At Champlain stack and further, we may start datanode as a non-root even in secure cluster
if not (params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0) or params.secure_dn_ports_are_in_use:
user = "******"
pid_file = format(
"{hadoop_pid_dir_prefix}/{hdfs_user}/hadoop-{hdfs_user}-{name}.pid")
if action == 'stop' and (params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0) and \
os.path.isfile(hadoop_secure_dn_pid_file):
# We need special handling for this case to handle the situation
# when we configure non-root secure DN and then restart it
# to handle new configs. Otherwise we will not be able to stop
# a running instance
user = "******"
try:
check_process_status(hadoop_secure_dn_pid_file)
custom_export = {
'HADOOP_SECURE_DN_USER': params.hdfs_user
}
hadoop_env_exports.update(custom_export)
except ComponentIsNotRunning:
pass
hadoop_daemon = format("{hadoop_bin}/hadoop-daemon.sh")
if user == "root":
cmd = [hadoop_daemon, "--config", params.hadoop_conf_dir, action, name]
if options:
cmd += [options, ]
daemon_cmd = as_sudo(cmd)
else:
cmd = format("{ulimit_cmd} {hadoop_daemon} --config {hadoop_conf_dir} {action} {name}")
if options:
cmd += " " + options
daemon_cmd = as_user(cmd, user)
if action == "start":
# remove pid file from dead process
File(pid_file, action="delete", not_if=process_id_exists_command)
Execute(daemon_cmd, not_if=process_id_exists_command, environment=hadoop_env_exports)
elif action == "stop":
Execute(daemon_cmd, only_if=process_id_exists_command, environment=hadoop_env_exports)
File(pid_file, action="delete")
def convert_conf_directories_to_symlinks(package, version, dirs, skip_existing_links=True, link_to="current"):
"""
Assumes HDP 2.3+, moves around directories and creates the conf symlink for the given package.
If the package does not exist, then no work is performed.
- Creates a /etc/<component>/conf.backup directory
- Copies all configs from /etc/<component>/conf to conf.backup
- Removes /etc/<component>/conf
- Creates /etc/<component>/<version>/0
- Creates /usr/hdp/current/<component>-client/conf -> /etc/<component>/<version>/0
- Links /etc/<component>/conf to <something>
-- /etc/<component>/conf -> /usr/hdp/current/[component]-client/conf
-- /etc/<component>/conf -> /etc/<component>/conf.backup
:param package: the package to create symlinks for (zookeeper, falcon, etc)
:param version: the version number to use with conf-select (2.3.0.0-1234)
:param dirs: the directories associated with the package (from PACKAGE_DIRS)
:param skip_existing_links: True to not do any work if already a symlink
:param link_to: link to "current" or "backup"
"""
bad_dirs = []
for dir_def in dirs:
if not os.path.exists(dir_def['conf_dir']):
bad_dirs.append(dir_def['conf_dir'])
if len(bad_dirs) > 0:
Logger.info("Skipping {0} as it does not exist.".format(",".join(bad_dirs)))
return
# existing links should be skipped since we assume there's no work to do
if skip_existing_links:
bad_dirs = []
for dir_def in dirs:
# check if conf is a link already
old_conf = dir_def['conf_dir']
if os.path.islink(old_conf):
Logger.info("{0} is already link to {1}".format(old_conf, os.path.realpath(old_conf)))
bad_dirs.append(old_conf)
if len(bad_dirs) > 0:
return
# make backup dir and copy everything in case configure() was called after install()
backup_dir = None
for dir_def in dirs:
old_conf = dir_def['conf_dir']
old_parent = os.path.abspath(os.path.join(old_conf, os.pardir))
backup_dir = os.path.join(old_parent, "conf.backup")
Logger.info("Backing up {0} to {1} if destination doesn't exist already.".format(old_conf, backup_dir))
Execute(("cp", "-R", "-p", old_conf, backup_dir),
not_if = format("test -e {backup_dir}"), sudo = True)
# we're already in the HDP stack
# Create the versioned /etc/[component]/[version]/0 folder.
# The component must be installed on the host.
versioned_confs = create("HDP", package, version, dry_run = True)
Logger.info("Package {0} will have new conf directories: {1}".format(package, ", ".join(versioned_confs)))
need_dirs = []
for d in versioned_confs:
if not os.path.exists(d):
need_dirs.append(d)
if len(need_dirs) > 0:
create("HDP", package, version)
# find the matching definition and back it up (not the most efficient way) ONLY if there is more than one directory
if len(dirs) > 1:
for need_dir in need_dirs:
for dir_def in dirs:
if 'prefix' in dir_def and need_dir.startswith(dir_def['prefix']):
old_conf = dir_def['conf_dir']
versioned_conf = need_dir
Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False),
only_if = format("ls -d {old_conf}/*"))
elif 1 == len(dirs) and 1 == len(need_dirs):
old_conf = dirs[0]['conf_dir']
versioned_conf = need_dirs[0]
Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False),
only_if = format("ls -d {old_conf}/*"))
# /usr/hdp/current/[component] is already set to to the correct version, e.g., /usr/hdp/[version]/[component]
link_component_conf_to_versioned_config(package, version)
# Symlink /etc/[component]/conf to /etc/[component]/conf.backup
try:
# No more references to /etc/[component]/conf
for dir_def in dirs:
# E.g., /etc/[component]/conf
new_symlink = dir_def['conf_dir']
# Remove new_symlink to pave the way, but only if it's a directory
if not os.path.islink(new_symlink):
Directory(new_symlink, action="delete")
if link_to in ["current", "backup"]:
# link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf
if link_to == "backup":
Link(new_symlink, to = backup_dir)
else:
Link(new_symlink, to = dir_def['current_dir'])
else:
Logger.error("Unsupported 'link_to' argument. Could not link package {0}".format(package))
except Exception, e:
Logger.warning("Could not change symlink for package {0} to point to {1} directory. Error: {2}".format(package, link_to, e))
def service(componentName, action='start', serviceName='yarn'):
import params
if serviceName == 'mapreduce' and componentName == 'historyserver':
delete_pid_file = True
daemon = format("{mapred_bin}/mr-jobhistory-daemon.sh")
pid_file = format(
"{mapred_pid_dir}/mapred-{mapred_user}-{componentName}.pid")
usr = params.mapred_user
log_dir = params.mapred_log_dir
else:
# !!! yarn-daemon.sh deletes the PID for us; if we remove it the script
# may not work correctly when stopping the service
delete_pid_file = False
daemon = format("{yarn_bin}/yarn-daemon.sh")
pid_file = format(
"{yarn_pid_dir}/yarn-{yarn_user}-{componentName}.pid")
usr = params.yarn_user
log_dir = params.yarn_log_dir
cmd = format(
"export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {hadoop_conf_dir}"
)
if action == 'start':
daemon_cmd = format("{ulimit_cmd} {cmd} start {componentName}")
check_process = as_sudo(["test", "-f", pid_file]) + " && " + as_sudo(
["pgrep", "-F", pid_file])
# Remove the pid file if its corresponding process is not running.
File(pid_file, action="delete", not_if=check_process)
if componentName == 'timelineserver' and serviceName == 'yarn':
File(params.ats_leveldb_lock_file,
action="delete",
only_if=format("ls {params.ats_leveldb_lock_file}"),
not_if=check_process,
ignore_failures=True)
try:
# Attempt to start the process. Internally, this is skipped if the process is already running.
Execute(daemon_cmd, user=usr, not_if=check_process)
# Ensure that the process with the expected PID exists.
Execute(
check_process,
not_if=check_process,
tries=5,
try_sleep=1,
)
except:
show_logs(log_dir, usr)
raise
elif action == 'stop':
daemon_cmd = format("{cmd} stop {componentName}")
try:
Execute(daemon_cmd, user=usr)
except:
show_logs(log_dir, usr)
raise
# !!! yarn-daemon doesn't need us to delete PIDs
if delete_pid_file is True:
File(pid_file, action="delete")
elif action == 'refreshQueues':
rm_kinit_cmd = params.rm_kinit_cmd
refresh_cmd = format(
"{rm_kinit_cmd} export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {yarn_container_bin}/yarn rmadmin -refreshQueues"
)
Execute(refresh_cmd, user=usr)
def convert_conf_directories_to_symlinks(package,
version,
dirs,
skip_existing_links=True,
link_to=DIRECTORY_TYPE_CURRENT):
"""
Assumes HDP 2.3+, moves around directories and creates the conf symlink for the given package.
If the package does not exist, then no work is performed.
- Creates a /etc/<component>/conf.backup directory
- Copies all configs from /etc/<component>/conf to conf.backup
- Removes /etc/<component>/conf
- Creates /etc/<component>/<version>/0 via <conf-selector-tool>
- <stack-root>/current/<component>-client/conf -> /etc/<component>/<version>/0 via <conf-selector-tool>
- Links /etc/<component>/conf to <something> depending on function paramter
-- /etc/<component>/conf -> <stack-root>/current/[component]-client/conf (usually)
-- /etc/<component>/conf -> /etc/<component>/conf.backup (only when supporting < HDP 2.3)
:param package: the package to create symlinks for (zookeeper, falcon, etc)
:param version: the version number to use with <conf-selector-tool> (2.3.0.0-1234)
:param dirs: the directories associated with the package (from get_package_dirs())
:param skip_existing_links: True to not do any work if already a symlink
:param link_to: link to "current" or "backup"
"""
# lack of enums makes this possible - we need to know what to link to
if link_to not in [DIRECTORY_TYPE_CURRENT, DIRECTORY_TYPE_BACKUP]:
raise Fail(
"Unsupported 'link_to' argument. Could not link package {0}".
format(package))
stack_name = Script.get_stack_name()
bad_dirs = []
for dir_def in dirs:
if not os.path.exists(dir_def['conf_dir']):
bad_dirs.append(dir_def['conf_dir'])
if len(bad_dirs) > 0:
Logger.info("Skipping {0} as it does not exist.".format(
",".join(bad_dirs)))
return
# existing links should be skipped since we assume there's no work to do
# they should be checked against the correct target though
if skip_existing_links:
bad_dirs = []
for dir_def in dirs:
# check if conf is a link already
old_conf = dir_def['conf_dir']
if os.path.islink(old_conf):
# it's already a link; make sure it's a link to where we want it
if link_to == DIRECTORY_TYPE_BACKUP:
target_conf_dir = _get_backup_conf_directory(old_conf)
else:
target_conf_dir = dir_def['current_dir']
# the link isn't to the right spot; re-link it
if os.readlink(old_conf) != target_conf_dir:
Logger.info("Re-linking symlink {0} to {1}".format(
old_conf, target_conf_dir))
Link(old_conf, action="delete")
Link(old_conf, to=target_conf_dir)
else:
Logger.info("{0} is already linked to {1}".format(
old_conf, os.path.realpath(old_conf)))
bad_dirs.append(old_conf)
if len(bad_dirs) > 0:
return
# make backup dir and copy everything in case configure() was called after install()
for dir_def in dirs:
old_conf = dir_def['conf_dir']
backup_dir = _get_backup_conf_directory(old_conf)
Logger.info(
"Backing up {0} to {1} if destination doesn't exist already.".
format(old_conf, backup_dir))
Execute(("cp", "-R", "-p", old_conf, backup_dir),
not_if=format("test -e {backup_dir}"),
sudo=True)
# we're already in the HDP stack
# Create the versioned /etc/[component]/[version]/0 folder.
# The component must be installed on the host.
versioned_confs = create(stack_name, package, version, dry_run=True)
Logger.info("Package {0} will have new conf directories: {1}".format(
package, ", ".join(versioned_confs)))
need_dirs = []
for d in versioned_confs:
if not os.path.exists(d):
need_dirs.append(d)
if len(need_dirs) > 0:
create(stack_name, package, version)
# find the matching definition and back it up (not the most efficient way) ONLY if there is more than one directory
if len(dirs) > 1:
for need_dir in need_dirs:
for dir_def in dirs:
if 'prefix' in dir_def and need_dir.startswith(
dir_def['prefix']):
old_conf = dir_def['conf_dir']
versioned_conf = need_dir
Execute(as_sudo([
"cp", "-R", "-p",
os.path.join(old_conf, "*"), versioned_conf
],
auto_escape=False),
only_if=format("ls -d {old_conf}/*"))
elif 1 == len(dirs) and 1 == len(need_dirs):
old_conf = dirs[0]['conf_dir']
versioned_conf = need_dirs[0]
Execute(as_sudo([
"cp", "-R", "-p",
os.path.join(old_conf, "*"), versioned_conf
],
auto_escape=False),
only_if=format("ls -d {old_conf}/*"))
# <stack-root>/current/[component] is already set to to the correct version, e.g., <stack-root>/[version]/[component]
select(stack_name, package, version, ignore_errors=True)
# Symlink /etc/[component]/conf to /etc/[component]/conf.backup
try:
# No more references to /etc/[component]/conf
for dir_def in dirs:
# E.g., /etc/[component]/conf
new_symlink = dir_def['conf_dir']
# Delete the existing directory/link so that linking will work
if not os.path.islink(new_symlink):
Directory(new_symlink, action="delete")
else:
Link(new_symlink, action="delete")
old_conf = dir_def['conf_dir']
backup_dir = _get_backup_conf_directory(old_conf)
# link /etc/[component]/conf -> /etc/[component]/conf.backup
# or
# link /etc/[component]/conf -> <stack-root>/current/[component]-client/conf
if link_to == DIRECTORY_TYPE_BACKUP:
Link(new_symlink, to=backup_dir)
else:
Link(new_symlink, to=dir_def['current_dir'])
#HACK
if package in [
"atlas",
]:
Logger.info(
"Seeding the new conf symlink {0} from the old backup directory {1} in case any "
"unmanaged artifacts are needed.".format(
new_symlink, backup_dir))
# If /etc/[component]/conf.backup exists, then copy any artifacts not managed by Ambari to the new symlink target
# Be careful not to clobber any existing files.
Execute(as_sudo([
"cp", "-R", "--no-clobber",
os.path.join(backup_dir, "*"), new_symlink
],
auto_escape=False),
only_if=format("test -e {new_symlink}"))
except Exception, e:
Logger.warning(
"Could not change symlink for package {0} to point to {1} directory. Error: {2}"
.format(package, link_to, e))
def _link_configs(package, version, dirs):
"""
Link a specific package's configuration directory
"""
bad_dirs = []
for dir_def in dirs:
if not os.path.exists(dir_def['conf_dir']):
bad_dirs.append(dir_def['conf_dir'])
if len(bad_dirs) > 0:
Logger.debug("Skipping {0} as it does not exist.".format(",".join(bad_dirs)))
return
bad_dirs = []
for dir_def in dirs:
# check if conf is a link already
old_conf = dir_def['conf_dir']
if os.path.islink(old_conf):
Logger.debug("{0} is a link to {1}".format(old_conf, os.path.realpath(old_conf)))
bad_dirs.append(old_conf)
if len(bad_dirs) > 0:
return
# make backup dir and copy everything in case configure() was called after install()
for dir_def in dirs:
old_conf = dir_def['conf_dir']
old_parent = os.path.abspath(os.path.join(old_conf, os.pardir))
old_conf_copy = os.path.join(old_parent, "conf.install")
Execute(("cp", "-R", "-p", old_conf, old_conf_copy),
not_if = format("test -e {old_conf_copy}"), sudo = True)
# we're already in the HDP stack
versioned_confs = conf_select.create("HDP", package, version, dry_run = True)
Logger.info("New conf directories: {0}".format(", ".join(versioned_confs)))
need_dirs = []
for d in versioned_confs:
if not os.path.exists(d):
need_dirs.append(d)
if len(need_dirs) > 0:
conf_select.create("HDP", package, version)
# find the matching definition and back it up (not the most efficient way) ONLY if there is more than one directory
if len(dirs) > 1:
for need_dir in need_dirs:
for dir_def in dirs:
if 'prefix' in dir_def and need_dir.startswith(dir_def['prefix']):
old_conf = dir_def['conf_dir']
versioned_conf = need_dir
Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False),
only_if = format("ls {old_conf}/*"))
elif 1 == len(dirs) and 1 == len(need_dirs):
old_conf = dirs[0]['conf_dir']
versioned_conf = need_dirs[0]
Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False),
only_if = format("ls {old_conf}/*"))
# make /usr/hdp/[version]/[component]/conf point to the versioned config.
# /usr/hdp/current is already set
try:
conf_select.select("HDP", package, version)
# no more references to /etc/[component]/conf
for dir_def in dirs:
Directory(dir_def['conf_dir'], action="delete")
# link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf
Link(dir_def['conf_dir'], to = dir_def['current_dir'])
except Exception, e:
Logger.warning("Could not select the directory: {0}".format(e.message))
def setup_kms_db():
import params
if params.has_ranger_admin:
password_validation(params.kms_master_key_password, 'KMS master key')
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
mode = 0644
)
Directory(params.java_share_dir,
mode=0755,
recursive=True,
cd_access="a"
)
if params.db_flavor.lower() != 'sqla':
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_curl_target, mode=0644)
Directory(os.path.join(params.kms_home, 'ews', 'lib'),
mode=0755
)
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(params.kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
Directory(params.jdbc_libs_dir,
cd_access="a",
recursive=True)
Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(params.kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(params.kms_home, 'ews', 'webapp', 'lib', params.jdbc_jar_name), mode=0644)
ModifyPropertiesFile(format("/usr/hdp/current/ranger-kms/install.properties"),
properties = params.config['configurations']['kms-properties'],
owner = params.kms_user
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{kms_home}/ews/webapp/lib/{jdbc_jar_name}')},
owner = params.kms_user,
)
env_dict = {'RANGER_KMS_HOME':params.kms_home, 'JAVA_HOME': params.java_home}
if params.db_flavor.lower() == 'sqla':
env_dict = {'RANGER_KMS_HOME':params.kms_home, 'JAVA_HOME': params.java_home, 'LD_LIBRARY_PATH':params.ld_library_path}
dba_setup = format('python {kms_home}/dba_script.py -q')
db_setup = format('python {kms_home}/db_setup.py')
if params.create_db_user:
Logger.info('Setting up Ranger KMS DB and DB User')
Execute(dba_setup, environment=env_dict, logoutput=True, user=params.kms_user, tries=5, try_sleep=10)
else:
Logger.info('Separate DBA property not set. Assuming Ranger KMS DB and DB User exists!')
Execute(db_setup, environment=env_dict, logoutput=True, user=params.kms_user, tries=5, try_sleep=10)
def convert_conf_directories_to_symlinks(package, version, dirs, skip_existing_links=True,
link_to=DIRECTORY_TYPE_CURRENT):
"""
Assumes HDP 2.3+, moves around directories and creates the conf symlink for the given package.
If the package does not exist, then no work is performed.
- Creates a /etc/<component>/conf.backup directory
- Copies all configs from /etc/<component>/conf to conf.backup
- Removes /etc/<component>/conf
- Creates /etc/<component>/<version>/0 via <conf-selector-tool>
- <stack-root>/current/<component>-client/conf -> /etc/<component>/<version>/0 via <conf-selector-tool>
- Links /etc/<component>/conf to <something> depending on function paramter
-- /etc/<component>/conf -> <stack-root>/current/[component]-client/conf (usually)
-- /etc/<component>/conf -> /etc/<component>/conf.backup (only when supporting < HDP 2.3)
:param package: the package to create symlinks for (zookeeper, falcon, etc)
:param version: the version number to use with <conf-selector-tool> (2.3.0.0-1234)
:param dirs: the directories associated with the package (from get_package_dirs())
:param skip_existing_links: True to not do any work if already a symlink
:param link_to: link to "current" or "backup"
"""
# lack of enums makes this possible - we need to know what to link to
if link_to not in [DIRECTORY_TYPE_CURRENT, DIRECTORY_TYPE_BACKUP]:
raise Fail("Unsupported 'link_to' argument. Could not link package {0}".format(package))
stack_name = Script.get_stack_name()
bad_dirs = []
for dir_def in dirs:
if not os.path.exists(dir_def['conf_dir']):
bad_dirs.append(dir_def['conf_dir'])
if len(bad_dirs) > 0:
Logger.info("Skipping {0} as it does not exist.".format(",".join(bad_dirs)))
return
# existing links should be skipped since we assume there's no work to do
# they should be checked against the correct target though
if skip_existing_links:
bad_dirs = []
for dir_def in dirs:
# check if conf is a link already
old_conf = dir_def['conf_dir']
if os.path.islink(old_conf):
# it's already a link; make sure it's a link to where we want it
if link_to == DIRECTORY_TYPE_BACKUP:
target_conf_dir = _get_backup_conf_directory(old_conf)
else:
target_conf_dir = dir_def['current_dir']
# the link isn't to the right spot; re-link it
if os.readlink(old_conf) != target_conf_dir:
Logger.info("Re-linking symlink {0} to {1}".format(old_conf, target_conf_dir))
Link(old_conf, action = "delete")
Link(old_conf, to = target_conf_dir)
else:
Logger.info("{0} is already linked to {1}".format(old_conf, os.path.realpath(old_conf)))
bad_dirs.append(old_conf)
if len(bad_dirs) > 0:
return
# make backup dir and copy everything in case configure() was called after install()
for dir_def in dirs:
old_conf = dir_def['conf_dir']
backup_dir = _get_backup_conf_directory(old_conf)
Logger.info("Backing up {0} to {1} if destination doesn't exist already.".format(old_conf, backup_dir))
Execute(("cp", "-R", "-p", old_conf, backup_dir),
not_if = format("test -e {backup_dir}"), sudo = True)
# we're already in the HDP stack
# Create the versioned /etc/[component]/[version]/0 folder.
# The component must be installed on the host.
versioned_confs = create(stack_name, package, version, dry_run = True)
Logger.info("Package {0} will have new conf directories: {1}".format(package, ", ".join(versioned_confs)))
need_dirs = []
for d in versioned_confs:
if not os.path.exists(d):
need_dirs.append(d)
if len(need_dirs) > 0:
create(stack_name, package, version)
# find the matching definition and back it up (not the most efficient way) ONLY if there is more than one directory
if len(dirs) > 1:
for need_dir in need_dirs:
for dir_def in dirs:
if 'prefix' in dir_def and need_dir.startswith(dir_def['prefix']):
old_conf = dir_def['conf_dir']
versioned_conf = need_dir
Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False),
only_if = format("ls -d {old_conf}/*"))
elif 1 == len(dirs) and 1 == len(need_dirs):
old_conf = dirs[0]['conf_dir']
versioned_conf = need_dirs[0]
Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False),
only_if = format("ls -d {old_conf}/*"))
# <stack-root>/current/[component] is already set to to the correct version, e.g., <stack-root>/[version]/[component]
select(stack_name, package, version, ignore_errors = True)
# Symlink /etc/[component]/conf to /etc/[component]/conf.backup
try:
# No more references to /etc/[component]/conf
for dir_def in dirs:
# E.g., /etc/[component]/conf
new_symlink = dir_def['conf_dir']
# Delete the existing directory/link so that linking will work
if not os.path.islink(new_symlink):
Directory(new_symlink, action = "delete")
else:
Link(new_symlink, action = "delete")
# link /etc/[component]/conf -> /etc/[component]/conf.backup
# or
# link /etc/[component]/conf -> <stack-root>/current/[component]-client/conf
if link_to == DIRECTORY_TYPE_BACKUP:
old_conf = dir_def['conf_dir']
backup_dir = _get_backup_conf_directory(old_conf)
Link(new_symlink, to = backup_dir)
else:
Link(new_symlink, to = dir_def['current_dir'])
except Exception, e:
Logger.warning("Could not change symlink for package {0} to point to {1} directory. Error: {2}".format(package, link_to, e))
def convert_conf_directories_to_symlinks(package,
version,
dirs,
skip_existing_links=True,
link_to="current"):
"""
Assumes HDP 2.3+, moves around directories and creates the conf symlink for the given package.
If the package does not exist, then no work is performed.
- Creates a /etc/<component>/conf.backup directory
- Copies all configs from /etc/<component>/conf to conf.backup
- Removes /etc/<component>/conf
- Creates /etc/<component>/<version>/0
- Creates /usr/hdp/current/<component>-client/conf -> /etc/<component>/<version>/0
- Links /etc/<component>/conf to <something>
-- /etc/<component>/conf -> /usr/hdp/current/[component]-client/conf
-- /etc/<component>/conf -> /etc/<component>/conf.backup
:param package: the package to create symlinks for (zookeeper, falcon, etc)
:param version: the version number to use with conf-select (2.3.0.0-1234)
:param dirs: the directories associated with the package (from PACKAGE_DIRS)
:param skip_existing_links: True to not do any work if already a symlink
:param link_to: link to "current" or "backup"
"""
bad_dirs = []
for dir_def in dirs:
if not os.path.exists(dir_def['conf_dir']):
bad_dirs.append(dir_def['conf_dir'])
if len(bad_dirs) > 0:
Logger.info("Skipping {0} as it does not exist.".format(
",".join(bad_dirs)))
return
# existing links should be skipped since we assume there's no work to do
if skip_existing_links:
bad_dirs = []
for dir_def in dirs:
# check if conf is a link already
old_conf = dir_def['conf_dir']
if os.path.islink(old_conf):
Logger.info("{0} is already linked to {1}".format(
old_conf, os.path.realpath(old_conf)))
bad_dirs.append(old_conf)
if len(bad_dirs) > 0:
return
# make backup dir and copy everything in case configure() was called after install()
backup_dir = None
for dir_def in dirs:
old_conf = dir_def['conf_dir']
old_parent = os.path.abspath(os.path.join(old_conf, os.pardir))
backup_dir = os.path.join(old_parent, "conf.backup")
Logger.info(
"Backing up {0} to {1} if destination doesn't exist already.".
format(old_conf, backup_dir))
Execute(("cp", "-R", "-p", old_conf, backup_dir),
not_if=format("test -e {backup_dir}"),
sudo=True)
# we're already in the HDP stack
# Create the versioned /etc/[component]/[version]/0 folder.
# The component must be installed on the host.
versioned_confs = create("HDP", package, version, dry_run=True)
Logger.info("Package {0} will have new conf directories: {1}".format(
package, ", ".join(versioned_confs)))
need_dirs = []
for d in versioned_confs:
if not os.path.exists(d):
need_dirs.append(d)
if len(need_dirs) > 0:
create("HDP", package, version)
# find the matching definition and back it up (not the most efficient way) ONLY if there is more than one directory
if len(dirs) > 1:
for need_dir in need_dirs:
for dir_def in dirs:
if 'prefix' in dir_def and need_dir.startswith(
dir_def['prefix']):
old_conf = dir_def['conf_dir']
versioned_conf = need_dir
Execute(as_sudo([
"cp", "-R", "-p",
os.path.join(old_conf, "*"), versioned_conf
],
auto_escape=False),
only_if=format("ls -d {old_conf}/*"))
elif 1 == len(dirs) and 1 == len(need_dirs):
old_conf = dirs[0]['conf_dir']
versioned_conf = need_dirs[0]
Execute(as_sudo([
"cp", "-R", "-p",
os.path.join(old_conf, "*"), versioned_conf
],
auto_escape=False),
only_if=format("ls -d {old_conf}/*"))
# /usr/hdp/current/[component] is already set to to the correct version, e.g., /usr/hdp/[version]/[component]
link_component_conf_to_versioned_config(package, version)
# Symlink /etc/[component]/conf to /etc/[component]/conf.backup
try:
# No more references to /etc/[component]/conf
for dir_def in dirs:
# E.g., /etc/[component]/conf
new_symlink = dir_def['conf_dir']
# Remove new_symlink to pave the way, but only if it's a directory
if not os.path.islink(new_symlink):
Directory(new_symlink, action="delete")
if link_to in ["current", "backup"]:
# link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf
if link_to == "backup":
Link(new_symlink, to=backup_dir)
else:
Link(new_symlink, to=dir_def['current_dir'])
else:
Logger.error(
"Unsupported 'link_to' argument. Could not link package {0}"
.format(package))
except Exception, e:
Logger.warning(
"Could not change symlink for package {0} to point to {1} directory. Error: {2}"
.format(package, link_to, e))
def service(componentName, action='start', serviceName='yarn'):
import params
if serviceName == 'mapreduce' and componentName == 'historyserver':
if not params.hdfs_tmp_dir or params.hdfs_tmp_dir == None or params.hdfs_tmp_dir.lower() == 'null':
Logger.error("WARNING: HDFS tmp dir property (hdfs_tmp_dir) is empty or invalid. Ambari will change permissions for the folder on regular basis.")
delete_pid_file = True
daemon = format("{mapred_bin}/mr-jobhistory-daemon.sh")
pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-{componentName}.pid")
usr = params.mapred_user
log_dir = params.mapred_log_dir
else:
# !!! yarn-daemon.sh deletes the PID for us; if we remove it the script
# may not work correctly when stopping the service
delete_pid_file = False
daemon = format("{yarn_bin}/yarn-daemon.sh")
pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-{componentName}.pid")
usr = params.yarn_user
log_dir = params.yarn_log_dir
cmd = format("export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {hadoop_conf_dir}")
if action == 'start':
# Check ambari-metrics-hadoop-sink version is less than 2.7.0.0
check_installed_metrics_hadoop_sink_version()
daemon_cmd = format("{ulimit_cmd} {cmd} start {componentName}")
check_process = as_sudo(["test", "-f", pid_file]) + " && " + as_sudo(["pgrep", "-F", pid_file])
# Remove the pid file if its corresponding process is not running.
File(pid_file, action = "delete", not_if = check_process)
if componentName == 'timelineserver' and serviceName == 'yarn':
File(params.ats_leveldb_lock_file,
action = "delete",
only_if = format("ls {params.ats_leveldb_lock_file}"),
not_if = check_process,
ignore_failures = True
)
try:
# Attempt to start the process. Internally, this is skipped if the process is already running.
Execute(daemon_cmd, user = usr, not_if = check_process)
# Ensure that the process with the expected PID exists.
Execute(check_process,
not_if = check_process,
tries=5,
try_sleep=1,
)
except:
show_logs(log_dir, usr)
raise
elif action == 'stop':
daemon_cmd = format("{cmd} stop {componentName}")
try:
Execute(daemon_cmd, user=usr)
except:
show_logs(log_dir, usr)
raise
# !!! yarn-daemon doesn't need us to delete PIDs
if delete_pid_file is True:
File(pid_file, action="delete")
elif action == 'refreshQueues':
rm_kinit_cmd = params.rm_kinit_cmd
refresh_cmd = format("{rm_kinit_cmd} export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {yarn_container_bin}/yarn rmadmin -refreshQueues")
Execute(refresh_cmd,
user = usr,
timeout = 20, # when Yarn is not started command hangs forever and should be killed
tries = 5,
try_sleep = 5,
timeout_kill_strategy = TerminateStrategy.KILL_PROCESS_GROUP, # the process cannot be simply killed by 'kill -15', so kill pg group instread.
)
def setup_ranger_db(rolling_upgrade=False):
import params
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source),
mode=0644)
Directory(params.java_share_dir, mode=0755, recursive=True, cd_access="a")
if params.db_flavor.lower() != 'sqla':
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_curl_target, mode=0644)
ranger_home = params.ranger_home
if rolling_upgrade:
ranger_home = format("/usr/hdp/{version}/ranger-admin")
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C',
params.tmp_dir),
sudo=True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive,
os.path.join(params.ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
Directory(params.jdbc_libs_dir, cd_access="a", recursive=True)
Execute(as_sudo([
'yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir
],
auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
os.path.join(params.ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(params.ranger_home, 'ews', 'lib', params.jdbc_jar_name),
mode=0644)
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties=params.config['configurations']['admin-properties'],
owner=params.unix_user,
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties={
'SQL_CONNECTOR_JAR':
format('{ranger_home}/ews/lib/{jdbc_jar_name}')
},
owner=params.unix_user,
)
env_dict = {
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home
}
if params.db_flavor.lower() == 'sqla':
env_dict = {
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home,
'LD_LIBRARY_PATH': params.ld_lib_path
}
# User wants us to setup the DB user and DB?
if params.create_db_dbuser:
Logger.info('Setting up Ranger DB and DB User')
dba_setup = format('ambari-python-wrap {ranger_home}/dba_script.py -q')
Execute(
dba_setup,
environment=env_dict,
logoutput=True,
user=params.unix_user,
)
else:
Logger.info(
'Separate DBA property not set. Assuming Ranger DB and DB User exists!'
)
db_setup = format('ambari-python-wrap {ranger_home}/db_setup.py')
Execute(
db_setup,
environment=env_dict,
logoutput=True,
user=params.unix_user,
)
def setup_ranger_db(rolling_upgrade=False):
import params
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
mode = 0644
)
Directory(params.java_share_dir,
mode=0755,
recursive=True,
cd_access="a"
)
if params.db_flavor.lower() != 'sqla':
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_curl_target, mode=0644)
ranger_home = params.ranger_home
if rolling_upgrade:
ranger_home = format("/usr/hdp/{version}/ranger-admin")
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(params.ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
Directory(params.jdbc_libs_dir,
cd_access="a",
recursive=True)
Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(params.ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(params.ranger_home, 'ews', 'lib',params.jdbc_jar_name), mode=0644)
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = params.config['configurations']['admin-properties'],
owner = params.unix_user,
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{ranger_home}/ews/lib/{jdbc_jar_name}')},
owner = params.unix_user,
)
env_dict = {'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME':params.java_home}
if params.db_flavor.lower() == 'sqla':
env_dict = {'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME':params.java_home, 'LD_LIBRARY_PATH':params.ld_lib_path}
# User wants us to setup the DB user and DB?
if params.create_db_dbuser:
Logger.info('Setting up Ranger DB and DB User')
dba_setup = format('ambari-python-wrap {ranger_home}/dba_script.py -q')
Execute(dba_setup,
environment=env_dict,
logoutput=True,
user=params.unix_user,
)
else:
Logger.info('Separate DBA property not set. Assuming Ranger DB and DB User exists!')
db_setup = format('ambari-python-wrap {ranger_home}/db_setup.py')
Execute(db_setup,
environment=env_dict,
logoutput=True,
user=params.unix_user,
)
def setup_ranger_admin(upgrade_type=None):
import params
check_db_connnection()
if params.driver_curl_source and not params.driver_curl_source.endswith("/None"):
if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar):
File(params.previous_jdbc_jar, action='delete')
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
mode = 0644
)
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_curl_target, mode=0644)
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = params.config['configurations']['admin-properties']
)
custom_config = dict()
custom_config['unix_user'] = params.unix_user
custom_config['unix_group'] = params.unix_group
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties=custom_config
)
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')}
)
##if db flavor == oracle - set oracle home env variable
if params.db_flavor.lower() == 'oracle' and params.oracle_home:
env_dict = {'JAVA_HOME': params.java_home, 'ORACLE_HOME':params.oracle_home, 'LD_LIBRARY_PATH':params.oracle_home}
else:
env_dict = {'JAVA_HOME': params.java_home}
setup_sh = format("cd {ranger_home} && ") + as_sudo([format('{ranger_home}/setup.sh')])
Execute(setup_sh,
environment=env_dict,
logoutput=True,
)
ModifyPropertiesFile(format("{ranger_conf}/xa_system.properties"),
properties = params.config['configurations']['ranger-site'],
)
ModifyPropertiesFile(format("{ranger_conf}/ranger_webserver.properties"),
properties = params.config['configurations']['ranger-site'],
mode=0744
)
Directory(params.admin_log_dir,
owner = params.unix_user,
group = params.unix_group
)
def setup_kms_db():
import params
if params.has_ranger_admin:
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
mode = 0644
)
Directory(params.java_share_dir,
mode=0755,
recursive=True,
cd_access="a"
)
if params.db_flavor.lower() != 'sqla':
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_curl_target, mode=0644)
Directory(os.path.join(params.kms_home, 'ews', 'lib'),
mode=0755
)
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(params.kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
Directory(params.jdbc_libs_dir,
cd_access="a",
recursive=True)
Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(params.kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(params.kms_home, 'ews', 'webapp', 'lib', params.jdbc_jar_name), mode=0644)
ModifyPropertiesFile(format("/usr/hdp/current/ranger-kms/install.properties"),
properties = params.config['configurations']['kms-properties'],
owner = params.kms_user
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{kms_home}/ews/webapp/lib/{jdbc_jar_name}')},
owner = params.kms_user,
)
env_dict = {'RANGER_KMS_HOME':params.kms_home, 'JAVA_HOME': params.java_home}
if params.db_flavor.lower() == 'sqla':
env_dict = {'RANGER_KMS_HOME':params.kms_home, 'JAVA_HOME': params.java_home, 'LD_LIBRARY_PATH':params.ld_library_path}
dba_setup = format('ambari-python-wrap {kms_home}/dba_script.py -q')
db_setup = format('ambari-python-wrap {kms_home}/db_setup.py')
Execute(dba_setup, environment=env_dict, logoutput=True, user=params.kms_user)
Execute(db_setup, environment=env_dict, logoutput=True, user=params.kms_user)
def _link_configs(package, version, dirs):
"""
Link a specific package's configuration directory
"""
bad_dirs = []
for dir_def in dirs:
if not os.path.exists(dir_def['conf_dir']):
bad_dirs.append(dir_def['conf_dir'])
if len(bad_dirs) > 0:
Logger.debug("Skipping {0} as it does not exist.".format(
",".join(bad_dirs)))
return
bad_dirs = []
for dir_def in dirs:
# check if conf is a link already
old_conf = dir_def['conf_dir']
if os.path.islink(old_conf):
Logger.debug("{0} is a link to {1}".format(
old_conf, os.path.realpath(old_conf)))
bad_dirs.append(old_conf)
if len(bad_dirs) > 0:
return
# make backup dir and copy everything in case configure() was called after install()
for dir_def in dirs:
old_conf = dir_def['conf_dir']
old_parent = os.path.abspath(os.path.join(old_conf, os.pardir))
old_conf_copy = os.path.join(old_parent, "conf.install")
Execute(("cp", "-R", "-p", old_conf, old_conf_copy),
not_if=format("test -e {old_conf_copy}"),
sudo=True)
# we're already in the HDP stack
versioned_confs = conf_select.create("HDP", package, version, dry_run=True)
Logger.info("New conf directories: {0}".format(", ".join(versioned_confs)))
need_dirs = []
for d in versioned_confs:
if not os.path.exists(d):
need_dirs.append(d)
if len(need_dirs) > 0:
conf_select.create("HDP", package, version)
# find the matching definition and back it up (not the most efficient way) ONLY if there is more than one directory
if len(dirs) > 1:
for need_dir in need_dirs:
for dir_def in dirs:
if 'prefix' in dir_def and need_dir.startswith(
dir_def['prefix']):
old_conf = dir_def['conf_dir']
versioned_conf = need_dir
Execute(as_sudo([
"cp", "-R", "-p",
os.path.join(old_conf, "*"), versioned_conf
],
auto_escape=False),
only_if=format("ls {old_conf}/*"))
elif 1 == len(dirs) and 1 == len(need_dirs):
old_conf = dirs[0]['conf_dir']
versioned_conf = need_dirs[0]
Execute(as_sudo([
"cp", "-R", "-p",
os.path.join(old_conf, "*"), versioned_conf
],
auto_escape=False),
only_if=format("ls {old_conf}/*"))
# make /usr/hdp/[version]/[component]/conf point to the versioned config.
# /usr/hdp/current is already set
try:
conf_select.select("HDP", package, version)
# no more references to /etc/[component]/conf
for dir_def in dirs:
Directory(dir_def['conf_dir'], action="delete")
# link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf
Link(dir_def['conf_dir'], to=dir_def['current_dir'])
except Exception, e:
Logger.warning("Could not select the directory: {0}".format(e.message))
