def jdbc_connector(): import params if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db: environment = {"no_proxy": format("{ambari_server_hostname}")} # TODO: should be removed after ranger_hive_plugin will not provide jdbc Execute(('rm', '-f', params.prepackaged_ojdbc_symlink), path=["/bin", "/usr/bin/"], sudo=True) File(params.downloaded_custom_connector, content=DownloadSource(params.driver_curl_source)) # maybe it will be more correcvly to use db type if params.sqla_db_used: untar_sqla_type2_driver = ('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir) Execute(untar_sqla_type2_driver, sudo=True) Execute(as_sudo([ 'yes', '|', 'cp', params.jars_path_in_archive, params.hive_lib ], auto_escape=False), path=["/bin", "/usr/bin/"]) Directory(params.jdbc_libs_dir, recursive=True) Execute(as_sudo([ 'yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir ], auto_escape=False), path=["/bin", "/usr/bin/"]) else: Execute( ('cp', '--remove-destination', params.downloaded_custom_connector, params.target), #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc path=["/bin", "/usr/bin/"], sudo=True) else: #for default hive db (Mysql) Execute( ('cp', '--remove-destination', format('/usr/share/java/{jdbc_jar_name}'), params.target), #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc path=["/bin", "/usr/bin/"], sudo=True) File( params.target, mode=0644, )
def enable_keep_alive(self): httpd_conf_dir = self.httpd_conf_dir command = format( "grep -E 'KeepAlive (On|Off)' {httpd_conf_dir}/httpd.conf && " + as_sudo(('sed', '-i', 's/KeepAlive Off/KeepAlive On/', format("{httpd_conf_dir}/httpd.conf"))) + " || echo 'KeepAlive On' | ") + as_sudo( ('tee', '--append', format('{httpd_conf_dir}/httpd.conf'))) + " > /dev/null" Execute(command)
def checkAndStopRegistyDNS(cmd): import params import status_params hadoop_env_exports = {'HADOOP_LIBEXEC_DIR': params.hadoop_libexec_dir} componentName = 'registrydns' daemon_cmd = format("{cmd} stop {componentName}") log_dir = params.yarn_log_dir # When registry dns is switched from non-privileged to privileged mode or the other way, # then the previous instance of registry dns has a different pid/user. # Checking if either of the processes are running and shutting them down if they are. for dns_pid_file, dns_user in [ (status_params.yarn_registry_dns_priv_pid_file, status_params.root_user), (status_params.yarn_registry_dns_pid_file, params.yarn_user) ]: process_id_exists_command = as_sudo([ "test", "-f", dns_pid_file ]) + " && " + as_sudo(["pgrep", "-F", dns_pid_file]) try: Execute(daemon_cmd, only_if=process_id_exists_command, user=dns_user) except: # When the registry dns port is modified but registry dns is not started # immediately, then the configs in yarn-env.sh & yarn-site.xml related # to registry dns may have already changed. This introduces a discrepancy # between the actual process that is running and the configs. # For example, when port is changed from 5300 to 53, # then dns port = 53 in yarn-site and YARN_REGISTRYDNS_SECURE_* envs in yarn-env.sh # are saved. So, while trying to shutdown the stray non-privileged registry dns process # after sometime, yarn daemon from the configs thinks that it needs privileged # access and throws an exception. In such cases, we try to kill the stray process. show_logs(log_dir, dns_user) pass process_id_does_not_exist_command = format( "! ( {process_id_exists_command} )") code, out = shell.call(process_id_does_not_exist_command, env=hadoop_env_exports, tries=5, try_sleep=5) if code != 0: code, out, err = shell.checked_call(("cat", dns_pid_file), sudo=True, env=hadoop_env_exports, stderr=subprocess32.PIPE) Logger.info("PID to kill was retrieved: '" + out + "'.") out = out.splitlines()[0] pid = out Execute(("kill", "-9", pid), sudo=True) File(dns_pid_file, action="delete")
def stop(self, env, upgrade_type=None): import params env.set_params(params) pid_file = getPid(params, self.nodeType) process_id_exists_command = as_sudo(["test", "-f", pid_file]) + " && " + as_sudo(["pgrep", "-F", pid_file]) daemon_cmd = get_daemon_cmd(params, self.nodeType, "stop") try: Execute(daemon_cmd, user=params.druid_user, only_if=process_id_exists_command, ) except: show_logs(params.druid_log_dir, params.druid_user) raise
def copy_jdbc_connector(kms_home): import params if params.jdbc_jar_name is None and params.driver_curl_source.endswith("/None"): error_message = "Error! Sorry, but we can't find jdbc driver related to {0} database to download from {1}. \ Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'".format(params.db_flavor, params.jdk_location) Logger.error(error_message) if params.driver_curl_source and not params.driver_curl_source.endswith("/None"): if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar): File(params.previous_jdbc_jar, action='delete') driver_curl_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}") File(params.downloaded_custom_connector, content = DownloadSource(params.driver_curl_source), mode = 0644 ) Directory(os.path.join(kms_home, 'ews', 'lib'), mode=0755 ) if params.db_flavor.lower() == 'sqla': Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True) Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(kms_home, 'ews', 'webapp', 'lib')), path=["/bin", "/usr/bin/"], sudo=True) Directory(params.jdbc_libs_dir, cd_access="a", create_parents=True) Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False), path=["/bin", "/usr/bin/"]) File(os.path.join(kms_home, 'ews', 'webapp', 'lib', 'sajdbc4.jar'), mode=0644) else: Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(kms_home, 'ews', 'webapp', 'lib')), path=["/bin", "/usr/bin/"], sudo=True) File(os.path.join(kms_home, 'ews', 'webapp', 'lib', params.jdbc_jar_name), mode=0644) ModifyPropertiesFile(format("{kms_home}/install.properties"), properties = params.config['configurations']['kms-properties'], owner = params.kms_user ) if params.db_flavor.lower() == 'sqla': ModifyPropertiesFile(format("{kms_home}/install.properties"), properties = {'SQL_CONNECTOR_JAR': format('{kms_home}/ews/webapp/lib/sajdbc4.jar')}, owner = params.kms_user, ) else: ModifyPropertiesFile(format("{kms_home}/install.properties"), properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')}, owner = params.kms_user, )
def link_config(old_conf, link_conf): """ Creates a config link following: 1. Checks if the old_conf location exists 2. If it does, check if it's a link already 3. Make a copy to /etc/[component]/conf.backup 4. Remove the old directory and create a symlink to link_conf :old_conf: the old config directory, ie /etc/[component]/config :link_conf: the new target for the config directory, ie /usr/hdp/current/[component-dir]/conf """ if not os.path.exists(old_conf): Logger.debug("Skipping {0}; it does not exist".format(old_conf)) return if os.path.islink(old_conf): Logger.debug("Skipping {0}; it is already a link".format(old_conf)) return old_parent = os.path.abspath(os.path.join(old_conf, os.pardir)) Logger.info("Linking {0} to {1}".format(old_conf, link_conf)) old_conf_copy = os.path.join(old_parent, "conf.backup") if not os.path.exists(old_conf_copy): Execute(as_sudo(["cp", "-R", "-p", old_conf, old_conf_copy]), logoutput=True) shutil.rmtree(old_conf, ignore_errors=True) # link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf Link(old_conf, to=link_conf)
def setup_usersync(upgrade_type=None): import params PropertiesFile(format("{usersync_home}/install.properties"), properties = params.config['configurations']['usersync-properties'], ) custom_config = dict() custom_config['unix_user'] = params.unix_user custom_config['unix_group'] = params.unix_group ModifyPropertiesFile(format("{usersync_home}/install.properties"), properties=custom_config ) cmd = format("cd {usersync_home} && ") + as_sudo([format('{usersync_home}/setup.sh')]) Execute(cmd, environment={'JAVA_HOME': params.java_home}, logoutput=True) File([params.usersync_start, params.usersync_stop], owner = params.unix_user ) File(params.usersync_services_file, mode = 0755, ) Directory(params.usersync_log_dir, owner = params.unix_user, group = params.unix_group )
def copy_jdbc_connector(stack_version=None): import params if params.jdbc_jar_name is None and params.driver_curl_source.endswith("/None"): error_message = format("{db_flavor} jdbc driver cannot be downloaded from {jdk_location}\nPlease run 'ambari-server setup --jdbc-db={db_flavor} --jdbc-driver={{path_to_jdbc}}' on ambari-server host.") raise Fail(error_message) if params.driver_curl_source and not params.driver_curl_source.endswith("/None"): if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar): File(params.previous_jdbc_jar, action='delete') File(params.downloaded_custom_connector, content = DownloadSource(params.driver_curl_source), mode = 0644 ) ranger_home = params.ranger_home if stack_version is not None: ranger_home = format("{stack_root}/{stack_version}/ranger-admin") driver_curl_target = format("{ranger_home}/ews/lib/{jdbc_jar_name}") if params.db_flavor.lower() == 'sqla': Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True) Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(ranger_home, 'ews', 'lib')), path=["/bin", "/usr/bin/"], sudo=True) File(os.path.join(ranger_home, 'ews', 'lib', 'sajdbc4.jar'), mode=0644) Directory(params.jdbc_libs_dir, cd_access="a", create_parents=True) Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False), path=["/bin", "/usr/bin/"]) else: Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(ranger_home, 'ews', 'lib')), path=["/bin", "/usr/bin/"], sudo=True) File(os.path.join(ranger_home, 'ews', 'lib',params.jdbc_jar_name), mode=0644) ModifyPropertiesFile(format("{ranger_home}/install.properties"), properties = params.config['configurations']['admin-properties'], owner = params.unix_user, ) if params.db_flavor.lower() == 'sqla': ModifyPropertiesFile(format("{ranger_home}/install.properties"), properties = {'SQL_CONNECTOR_JAR': format('{ranger_home}/ews/lib/sajdbc4.jar')}, owner = params.unix_user, ) else: ModifyPropertiesFile(format("{ranger_home}/install.properties"), properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')}, owner = params.unix_user, )
def hbase_service(name, action='start'): # 'start' or 'stop' or 'status' import params role = name cmd = format("{daemon_script} --config {hbase_conf_dir}") pid_file = format("{pid_dir}/hbase-{hbase_user}-{role}.pid") pid_expression = as_sudo(["cat", pid_file]) no_op_test = as_sudo([ "test", "-f", pid_file ]) + format(" && ps -p `{pid_expression}` >/dev/null 2>&1") if action == 'start': # Check ambari-metrics-hadoop-sink version is less than 2.7.0.0 check_installed_metrics_hadoop_sink_version() daemon_cmd = format("{cmd} start {role}") try: Execute(daemon_cmd, not_if=no_op_test, user=params.hbase_user) except: show_logs(params.log_dir, params.hbase_user) raise elif action == 'stop': daemon_cmd = format("{cmd} stop {role}") try: Execute( daemon_cmd, user=params.hbase_user, only_if=no_op_test, # BUGFIX: hbase regionserver sometimes hangs when nn is in safemode timeout=params.hbase_regionserver_shutdown_timeout, on_timeout=format( "! ( {no_op_test} ) || {sudo} -H -E kill -9 `{pid_expression}`" ), ) except: show_logs(params.log_dir, params.hbase_user) raise File( pid_file, action="delete", )
def start(self, env, upgrade_type=None): import params env.set_params(params) install_druid() self.configure(env, upgrade_type=upgrade_type) daemon_cmd = get_daemon_cmd(params, self.nodeType, "start") # Verify Database connection on Druid start if params.metadata_storage_type == 'mysql': if not params.jdbc_driver_jar or not os.path.isfile( params.connector_download_dir + os.path.sep + params.jdbc_driver_jar): path_to_jdbc = params.connector_download_dir + os.path.sep + "*" error_message = "Error! Sorry, but we can't find jdbc driver for mysql.So, db connection check can fail." + \ "Please run 'ambari-server setup --jdbc-db=mysql --jdbc-driver={path_to_jdbc} on server host.'" Logger.error(error_message) else: path_to_jdbc = params.connector_download_dir + os.path.sep + params.jdbc_driver_jar db_connection_check_command = format( "{params.java8_home}/bin/java -cp {params.check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{params.metadata_storage_url}' {params.metadata_storage_user} {params.metadata_storage_password!p} com.mysql.jdbc.Driver" ) else: db_connection_check_command = None if db_connection_check_command: sudo.chmod(params.check_db_connection_jar, 0755) Execute( db_connection_check_command, tries=5, try_sleep=10, user=params.druid_user) pid_file = getPid(params, self.nodeType) process_id_exists_command = as_sudo([ "test", "-f", pid_file ]) + " && " + as_sudo(["pgrep", "-F", pid_file]) try: Execute( daemon_cmd, user=params.druid_user, not_if=process_id_exists_command, ) except: show_logs(params.druid_log_dir, params.druid_user) raise
def hbase_service(name, action='start'): # 'start' or 'stop' import params sudo = AMBARI_SUDO_BINARY daemon_script = format("{yarn_hbase_bin}/hbase-daemon.sh") role = name cmd = format("{daemon_script} --config {yarn_hbase_conf_dir}") pid_file = format( "{yarn_hbase_pid_dir}/hbase-{yarn_hbase_user}-{role}.pid") pid_expression = as_sudo(["cat", pid_file]) no_op_test = as_sudo([ "test", "-f", pid_file ]) + format(" && ps -p `{pid_expression}` >/dev/null 2>&1") if action == 'start': daemon_cmd = format("{cmd} start {role}") try: Execute(daemon_cmd, not_if=no_op_test, user=params.yarn_hbase_user) except: show_logs(params.yarn_hbase_log_dir, params.yarn_hbase_user) raise elif action == 'stop': daemon_cmd = format("{cmd} stop {role}") try: Execute( daemon_cmd, user=params.yarn_hbase_user, only_if=no_op_test, timeout=30, on_timeout=format( "! ( {no_op_test} ) || {sudo} -H -E kill -9 `{pid_expression}`" ), ) except: show_logs(params.yarn_hbase_log_dir, params.yarn_hbase_user) raise File( pid_file, action="delete", )
def setup_hdp_install_directory(): # This is a name of marker file. SELECT_ALL_PERFORMED_MARKER = "/var/lib/ambari-agent/data/hdp-select-set-all.performed" import params if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0: Execute(as_sudo(['touch', SELECT_ALL_PERFORMED_MARKER]) + ' ; ' + format('{sudo} /usr/bin/hdp-select set all `ambari-python-wrap /usr/bin/hdp-select versions | grep ^{stack_version_unformatted} | tail -1`'), only_if=format('ls -d /usr/hdp/{stack_version_unformatted}*'), # If any HDP version is installed not_if=format("test -f {SELECT_ALL_PERFORMED_MARKER}") # Do that only once (otherwise we break rolling upgrade logic) )
def stop(self, env): import params env.set_params(params) self.configure(env) daemon_cmd = self.get_daemon_cmd(params, "stop") pid_file = params.superset_pid_dir + '/superset.pid' process_id_exists_command = as_sudo([ "test", "-f", pid_file ]) + " && " + as_sudo(["pgrep", "-F", pid_file]) try: Execute( daemon_cmd, user=params.superset_user, only_if=process_id_exists_command, ) except: show_logs(params.superset_log_dir, params.superset_user) raise
def hbase_service(name, action='start'): import params role = name cmd = format("{daemon_script} --config {hbase_conf_dir}") pid_file = format("{pid_dir}/hbase-{hbase_user}-{role}.pid") pid_expression = as_sudo(["cat", pid_file]) no_op_test = as_sudo([ "test", "-f", pid_file ]) + format(" && ps -p `{pid_expression}` >/dev/null 2>&1") # delete wal log if HBase version has moved down if params.to_backup_wal_dir: wal_directory = params.wal_directory timestamp = datetime.now() timestamp_format = '%Y%m%d%H%M%S' wal_directory_backup = '%s_%s' % (wal_directory, timestamp.strftime(timestamp_format)) check_if_wal_dir_exists = format("hdfs dfs -ls {wal_directory}") wal_dir_exists = False try: Execute(check_if_wal_dir_exists, user=params.hbase_user) wal_dir_exists = True except Exception, e: Logger.error( format( "Did not find HBase WAL directory {wal_directory}. It's possible that it was already moved. Exception: {e.message}" )) if wal_dir_exists: move_wal_dir_cmd = format( "hdfs dfs -mv {wal_directory} {wal_directory_backup}") try: Execute(move_wal_dir_cmd, user=params.hbase_user) except Exception, e: Logger.error( format( "Failed to backup HBase WAL directory, command: {move_wal_dir_cmd} . Exception: {e.message}" ))
def copy_jdbc_connector(stack_version=None): import params File(params.downloaded_custom_connector, content = DownloadSource(params.driver_curl_source), mode = 0644 ) ranger_home = params.ranger_home if stack_version is not None: ranger_home = format("{stack_root}/{stack_version}/ranger-admin") if params.db_flavor.lower() == 'sqla': Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True) Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(ranger_home, 'ews', 'lib')), path=["/bin", "/usr/bin/"], sudo=True) File(os.path.join(ranger_home, 'ews', 'lib', 'sajdbc4.jar'), mode=0644) Directory(params.jdbc_libs_dir, cd_access="a", create_parents=True) Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False), path=["/bin", "/usr/bin/"]) else: Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(ranger_home, 'ews', 'lib')), path=["/bin", "/usr/bin/"], sudo=True) File(os.path.join(ranger_home, 'ews', 'lib',params.jdbc_jar_name), mode=0644) ModifyPropertiesFile(format("{ranger_home}/install.properties"), properties = params.config['configurations']['admin-properties'], owner = params.unix_user, ) if params.db_flavor.lower() == 'sqla': ModifyPropertiesFile(format("{ranger_home}/install.properties"), properties = {'SQL_CONNECTOR_JAR': format('{ranger_home}/ews/lib/sajdbc4.jar')}, owner = params.unix_user, ) else: ModifyPropertiesFile(format("{ranger_home}/install.properties"), properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')}, owner = params.unix_user, )
def setup_ranger_admin(upgrade_type=None): import params check_db_connnection() File(params.driver_curl_target, mode=0644) ModifyPropertiesFile(format("{ranger_home}/install.properties"), properties=params.config['configurations']['admin-properties'] ) custom_config = dict() custom_config['unix_user'] = params.unix_user custom_config['unix_group'] = params.unix_group ModifyPropertiesFile(format("{ranger_home}/install.properties"), properties=custom_config ) ModifyPropertiesFile(format("{ranger_home}/install.properties"), properties={'SQL_CONNECTOR_JAR': format('{driver_curl_target}')} ) ##if db flavor == oracle - set oracle home env variable if params.db_flavor.lower() == 'oracle' and params.oracle_home: env_dict = {'JAVA_HOME': params.java_home, 'ORACLE_HOME': params.oracle_home, 'LD_LIBRARY_PATH': params.oracle_home} else: env_dict = {'JAVA_HOME': params.java_home} setup_sh = format("cd {ranger_home} && ") + as_sudo([format('{ranger_home}/setup.sh')]) Execute(setup_sh, environment=env_dict, logoutput=True, ) ModifyPropertiesFile(format("{ranger_conf}/xa_system.properties"), properties=params.config['configurations']['ranger-site'], ) ModifyPropertiesFile(format("{ranger_conf}/ranger_webserver.properties"), properties=params.config['configurations']['ranger-site'], mode=0744 ) Directory(params.admin_log_dir, owner=params.unix_user, group=params.unix_group )
def _copy_configurations(source_directory, target_directory): """ Copies from the source directory to the target directory. If the source directory is a symlink then it will be followed (deferenced) but any other symlinks found to copy will not be. This will ensure that if the configuration directory itself is a symlink, then it's contents will be copied, preserving and children found which are also symlinks. :param source_directory: the source directory to copy from :param target_directory: the target directory to copy to :return: None """ # append trailing slash so the cp command works correctly WRT recursion and symlinks source_directory = os.path.join(source_directory, "*") Execute(as_sudo(["cp", "-R", "-p", "-v", source_directory, target_directory], auto_escape = False), logoutput = True)
def create_30_config_version(self, env): package_name = 'registry' stack_root = Script.get_stack_root() current_dir = "{0}/current/registry/conf".format(stack_root) directories = [{ "conf_dir": "/etc/registry/conf", "current_dir": current_dir }] stack_version = stack_select.get_stack_version_before_install( package_name) conf_dir = "/etc/registry/conf" if stack_version: try: #Check if broken symbolic links issue exists os.stat(conf_dir) conf_select.convert_conf_directories_to_symlinks( package_name, stack_version, directories) cp_cmd = as_sudo([ "cp", "-a", "-f", "/etc/registry/conf.backup/.", "/etc/registry/conf" ]) Execute(cp_cmd, logoutput=True) except OSError as e: Logger.warning( "Detected broken symlink : {0}. Attempting to repair.". format(str(e))) #removing symlink conf directory sudo.unlink(conf_dir) #make conf dir again sudo.makedirs(conf_dir, 0755) #copy all files for files in glob.glob("/etc/registry/conf.backup/*"): cp_cmd = as_sudo(["cp", "-r", files, conf_dir]) Execute(cp_cmd, logoutput=True) conf_select.convert_conf_directories_to_symlinks( package_name, stack_version, directories)
def _link_configs(package, version, old_conf, link_conf): """ Link a specific package's configuration directory """ if not os.path.exists(old_conf): Logger.debug("Skipping {0} as it does not exist.".format(old_conf)) return # check if conf is a link to the target already if os.path.islink(old_conf): Logger.debug("{0} is already a link to {1}".format(old_conf, os.path.realpath(old_conf))) return # make backup dir and copy everything in case configure() was called after install() old_parent = os.path.abspath(os.path.join(old_conf, os.pardir)) old_conf_copy = os.path.join(old_parent, "conf.install") Execute(("cp", "-R", "-p", old_conf, old_conf_copy), not_if = format("test -e {old_conf_copy}"), sudo = True, ) versioned_conf = conf_select.create("HDP", package, version, dry_run = True) Logger.info("New conf directory is {0}".format(versioned_conf)) # make new conf dir and copy everything in case configure() was called after install() if not os.path.exists(versioned_conf): conf_select.create("HDP", package, version) Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False), only_if = format("ls {old_conf}/*") ) # make /usr/hdp/<version>/hadoop/conf point to the versioned config. # /usr/hdp/current is already set conf_select.select("HDP", package, version) # no more references to /etc/[component]/conf Directory(old_conf, action="delete", ) # link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf Link(old_conf, to = link_conf )
def link_config(old_conf, link_conf): """ Creates a config link following: 1. Checks if the old_conf location exists 2. If it does, check if it's a link already 3. Make a copy to /etc/[component]/conf.backup 4. Remove the old directory and create a symlink to link_conf :old_conf: the old config directory, ie /etc/[component]/conf :link_conf: the new target for the config directory, ie /usr/hdp/current/[component-dir]/conf """ # if the link exists but is wrong, then change it if os.path.islink(old_conf) and os.path.realpath(old_conf) != link_conf: Link(old_conf, to = link_conf) return if os.path.islink(old_conf): Logger.debug("Skipping {0}; it is already a link".format(old_conf)) return if not os.path.exists(old_conf): Logger.debug("Skipping {0}; it does not exist".format(old_conf)) return old_parent = os.path.abspath(os.path.join(old_conf, os.pardir)) Logger.info("Linking {0} to {1}".format(old_conf, link_conf)) old_conf_copy = os.path.join(old_parent, "conf.backup") if not os.path.exists(old_conf_copy): Execute(as_sudo(["cp", "-R", "-p", old_conf, old_conf_copy]), logoutput=True) shutil.rmtree(old_conf, ignore_errors=True) # link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf Link(old_conf, to = link_conf)
def enable_keep_alive(self): httpd_conf_dir = self.httpd_conf_dir command = format("grep -E 'KeepAlive (On|Off)' {httpd_conf_dir}/httpd.conf && " + as_sudo(('sed', '-i','s/KeepAlive Off/KeepAlive On/', format("{httpd_conf_dir}/httpd.conf"))) + " || echo 'KeepAlive On' | ") + as_sudo(('tee', '--append', format('{httpd_conf_dir}/httpd.conf'))) + " > /dev/null" Execute(command )
def spark_service(name, upgrade_type=None, action=None): import params if action == 'start': check_sac_jar() effective_version = params.version if upgrade_type is not None else params.stack_version_formatted if effective_version: effective_version = format_stack_version(effective_version) if name == 'jobhistoryserver' and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version): # create & copy spark2-hdp-yarn-archive.tar.gz to hdfs if not params.sysprep_skip_copy_tarballs_hdfs: source_dirs = [params.spark_home + "/jars"] # include sac jar and spark-job keytab to archive if params.sac_enabled: if params.security_enabled: shutil.copy(params.atlas_kafka_keytab, source_dirs[0]) os.chmod(os.path.join(source_dirs[0], os.path.basename(params.atlas_kafka_keytab)), 0440) source_dirs.append(params.spark_atlas_jar_dir) tmp_archive_file=get_tarball_paths("spark2")[1] make_tarfile(tmp_archive_file, source_dirs) copy_to_hdfs("spark2", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs, replace_existing_files=True) if params.sac_enabled and params.security_enabled: os.remove(os.path.join(source_dirs[0], os.path.basename(params.atlas_kafka_keytab))) # create & copy spark2-hdp-hive-archive.tar.gz to hdfs if not params.sysprep_skip_copy_tarballs_hdfs: source_dirs = [params.spark_home+"/standalone-metastore"] tmp_archive_file=get_tarball_paths("spark2hive")[1] make_tarfile(tmp_archive_file, source_dirs) copy_to_hdfs("spark2hive", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs, replace_existing_files=True) # create spark history directory params.HdfsResource(params.spark_history_dir, type="directory", action="create_on_execute", owner=params.spark_user, group=params.user_group, mode=0777, recursive_chmod=True ) params.HdfsResource(None, action="execute") if params.security_enabled: spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ") Execute(spark_kinit_cmd, user=params.spark_user) # Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not # need to copy the tarball, otherwise, copy it. if params.stack_version_formatted and check_stack_feature(StackFeature.TEZ_FOR_SPARK, params.stack_version_formatted): resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs) if resource_created: params.HdfsResource(None, action="execute") if name == 'jobhistoryserver': create_catalog_cmd = format("{hive_schematool_bin}/schematool -dbType {hive_metastore_db_type} " "-createCatalog {default_metastore_catalog} " "-catalogDescription 'Default catalog, for Spark' -ifNotExists " "-catalogLocation {default_fs}{spark_warehouse_dir}") Execute(create_catalog_cmd, user = params.hive_user) historyserver_no_op_test = as_sudo(["test", "-f", params.spark_history_server_pid_file]) + " && " + as_sudo(["pgrep", "-F", params.spark_history_server_pid_file]) try: Execute(params.spark_history_server_start, user=params.spark_user, environment={'JAVA_HOME': params.java_home}, not_if=historyserver_no_op_test) except: show_logs(params.spark_log_dir, user=params.spark_user) raise elif name == 'sparkthriftserver': import status_params if params.security_enabled: hive_kinit_cmd = format("{kinit_path_local} -kt {hive_kerberos_keytab} {hive_kerberos_principal}; ") Execute(hive_kinit_cmd, user=params.spark_user) thriftserver_no_op_test= as_sudo(["test", "-f", params.spark_thrift_server_pid_file]) + " && " + as_sudo(["pgrep", "-F", params.spark_thrift_server_pid_file]) try: Execute(format('{spark_thrift_server_start} --properties-file {spark_thrift_server_conf_file} {spark_thrift_cmd_opts_properties}'), user=params.spark_user, environment={'JAVA_HOME': params.java_home}, not_if=thriftserver_no_op_test ) except: show_logs(params.spark_log_dir, user=params.spark_user) raise hive_connection_created = False i = 0 while i < 15: time.sleep(30) Logger.info("Check connection to STS is created.") beeline_url = ["jdbc:hive2://{fqdn}:{spark_thrift_port}/default"] if params.security_enabled: beeline_url.append("principal={hive_kerberos_principal}") beeline_url.append("transportMode={spark_transport_mode}") if params.spark_transport_mode.lower() == 'http': beeline_url.append("httpPath={spark_thrift_endpoint}") if params.spark_thrift_ssl_enabled: beeline_url.append("ssl=true") beeline_cmd = os.path.join(params.spark_home, "bin", "beeline") cmd = "! %s -u '%s' -e '' 2>&1| awk '{print}'|grep -i -e 'Connection refused' -e 'Invalid URL' -e 'Error: Could not open'" % \ (beeline_cmd, format(";".join(beeline_url))) try: Execute(cmd, user=params.spark_user, path=[beeline_cmd], timeout=CHECK_COMMAND_TIMEOUT_DEFAULT) hive_connection_created = True Logger.info("Connection to STS is created.") break except: Logger.info("Connection to STS still is not created.") pass Logger.info("Check STS process status.") check_process_status(status_params.spark_thrift_server_pid_file) i+=1 if not hive_connection_created: raise ComponentIsNotRunning("Something goes wrong, STS connection was not created but STS process still alive. " "Potential problems: Hive/YARN doesn't work correctly or too slow. For more information check STS logs.") elif action == 'stop': if name == 'jobhistoryserver': try: Execute(format('{spark_history_server_stop}'), user=params.spark_user, environment={'JAVA_HOME': params.java_home} ) except: show_logs(params.spark_log_dir, user=params.spark_user) raise File(params.spark_history_server_pid_file, action="delete" ) elif name == 'sparkthriftserver': try: Execute(format('{spark_thrift_server_stop}'), user=params.spark_user, environment={'JAVA_HOME': params.java_home} ) except: show_logs(params.spark_log_dir, user=params.spark_user) raise File(params.spark_thrift_server_pid_file, action="delete" )
def service(action=None, name=None, user=None, options="", create_pid_dir=False, create_log_dir=False): """ :param action: Either "start" or "stop" :param name: Component name, e.g., "namenode", "datanode", "secondarynamenode", "zkfc" :param user: User to run the command as :param options: Additional options to pass to command as a string :param create_pid_dir: Create PID directory :param create_log_dir: Crate log file directory """ import params options = options if options else "" pid_dir = format("{hadoop_pid_dir_prefix}/{user}") pid_file = format("{pid_dir}/hadoop-{user}-{name}.pid") hadoop_env_exports = {'HADOOP_LIBEXEC_DIR': params.hadoop_libexec_dir} log_dir = format("{hdfs_log_dir_prefix}/{user}") # NFS GATEWAY is always started by root using jsvc due to rpcbind bugs # on Linux such as CentOS6.2. https://bugzilla.redhat.com/show_bug.cgi?id=731542 if name == "nfs3": pid_file = format("{pid_dir}/hadoop_privileged_nfs3.pid") custom_export = { 'HADOOP_PRIVILEGED_NFS_USER': params.hdfs_user, 'HADOOP_PRIVILEGED_NFS_PID_DIR': pid_dir, 'HADOOP_PRIVILEGED_NFS_LOG_DIR': log_dir } hadoop_env_exports.update(custom_export) process_id_exists_command = as_sudo( ["test", "-f", pid_file]) + " && " + as_sudo(["pgrep", "-F", pid_file]) # on STOP directories shouldn't be created # since during stop still old dirs are used (which were created during previous start) if action != "stop": if name == "nfs3": Directory(params.hadoop_pid_dir_prefix, mode=0755, owner=params.root_user, group=params.root_group) else: Directory(params.hadoop_pid_dir_prefix, mode=0755, owner=params.hdfs_user, group=params.user_group) if create_pid_dir: Directory(pid_dir, owner=user, group=params.user_group, create_parents=True) if create_log_dir: if name == "nfs3": Directory(log_dir, mode=0775, owner=params.root_user, group=params.user_group) else: Directory(log_dir, owner=user, group=params.user_group, create_parents=True) if params.security_enabled and name == "datanode": ## The directory where pid files are stored in the secure data environment. hadoop_secure_dn_pid_dir = format( "{hadoop_pid_dir_prefix}/{hdfs_user}") hadoop_secure_dn_pid_file = format( "{hadoop_secure_dn_pid_dir}/hadoop_secure_dn.pid") # At datanode_non_root stack version and further, we may start datanode as a non-root even in secure cluster if not (params.stack_version_formatted and check_stack_feature( StackFeature.DATANODE_NON_ROOT, params.stack_version_formatted) ) or params.secure_dn_ports_are_in_use: user = "******" pid_file = format( "{hadoop_pid_dir_prefix}/{hdfs_user}/hadoop-{hdfs_user}-{name}.pid" ) if action == 'stop' and (params.stack_version_formatted and check_stack_feature(StackFeature.DATANODE_NON_ROOT, params.stack_version_formatted)) and \ os.path.isfile(hadoop_secure_dn_pid_file): # We need special handling for this case to handle the situation # when we configure non-root secure DN and then restart it # to handle new configs. Otherwise we will not be able to stop # a running instance user = "******" try: check_process_status(hadoop_secure_dn_pid_file) custom_export = {'HADOOP_SECURE_DN_USER': params.hdfs_user} hadoop_env_exports.update(custom_export) except ComponentIsNotRunning: pass hdfs_bin = format("{hadoop_bin}/hdfs") if user == "root": cmd = [ hdfs_bin, "--config", params.hadoop_conf_dir, "--daemon", action, name ] if options: cmd += [ options, ] daemon_cmd = as_sudo(cmd) else: cmd = format( "{ulimit_cmd} {hdfs_bin} --config {hadoop_conf_dir} --daemon {action} {name}" ) if options: cmd += " " + options daemon_cmd = as_user(cmd, user) if action == "start": # remove pid file from dead process File(pid_file, action="delete", not_if=process_id_exists_command) try: Execute(daemon_cmd, not_if=process_id_exists_command, environment=hadoop_env_exports) except: show_logs(log_dir, user) raise elif action == "stop": try: Execute(daemon_cmd, only_if=process_id_exists_command, environment=hadoop_env_exports) except: show_logs(log_dir, user) raise wait_process_stopped(pid_file) File(pid_file, action="delete")
def service(componentName, action='start', serviceName='yarn'): import params import status_params if serviceName == 'mapreduce' and componentName == 'historyserver': delete_pid_file = True daemon = format("{mapred_bin}/mapred") pid_file = format( "{mapred_pid_dir}/hadoop-{mapred_user}-{componentName}.pid") usr = params.mapred_user log_dir = params.mapred_log_dir cmd = format( "export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --daemon" ) else: # !!! yarn-daemon.sh deletes the PID for us; if we remove it the script # may not work correctly when stopping the service delete_pid_file = True daemon = format("{yarn_bin}/yarn") if componentName == 'registrydns' and status_params.registry_dns_needs_privileged_access: pid_file = status_params.yarn_registry_dns_priv_pid_file usr = status_params.root_user else: pid_file = format( "{yarn_pid_dir}/hadoop-{yarn_user}-{componentName}.pid") usr = params.yarn_user log_dir = params.yarn_log_dir cmd = format( "export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {hadoop_conf_dir} --daemon" ) check_process = as_sudo(["test", "-f", pid_file]) + " && " + as_sudo( ["pgrep", "-F", pid_file]) if action == 'start': daemon_cmd = format("{ulimit_cmd} {cmd} start {componentName}") if componentName == 'registrydns': checkAndStopRegistyDNS(cmd) else: # Remove the pid file if its corresponding process is not running. File(pid_file, action="delete", not_if=check_process) if componentName == 'timelineserver' and serviceName == 'yarn': File(params.ats_leveldb_lock_file, action="delete", only_if=format("ls {params.ats_leveldb_lock_file}"), not_if=check_process, ignore_failures=True) try: # Attempt to start the process. Internally, this is skipped if the process is already running. Execute(daemon_cmd, user=usr, not_if=check_process) # Ensure that the process with the expected PID exists. Execute( check_process, not_if=check_process, tries=5, try_sleep=1, ) except: show_logs(log_dir, usr) raise elif action == 'stop': daemon_cmd = format("{cmd} stop {componentName}") if componentName == 'registrydns': checkAndStopRegistyDNS(cmd) else: try: Execute(daemon_cmd, user=usr, only_if=check_process) except: show_logs(log_dir, usr) raise # !!! yarn-daemon doesn't need us to delete PIDs if delete_pid_file is True: File(pid_file, action="delete") elif action == 'refreshQueues': rm_kinit_cmd = params.rm_kinit_cmd refresh_cmd = format( "{rm_kinit_cmd} export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {yarn_bin}/yarn rmadmin -refreshQueues" ) Execute( refresh_cmd, user=usr, timeout= 20, # when Yarn is not started command hangs forever and should be killed tries=5, try_sleep=5, timeout_kill_strategy=TerminateStrategy. KILL_PROCESS_GROUP, # the process cannot be simply killed by 'kill -15', so kill pg group instread. )
def service(action=None, name=None, user=None, options="", create_pid_dir=False, create_log_dir=False): """ :param action: Either "start" or "stop" :param name: Component name, e.g., "namenode", "datanode", "secondarynamenode", "zkfc" :param user: User to run the command as :param options: Additional options to pass to command as a string :param create_pid_dir: Create PID directory :param create_log_dir: Crate log file directory """ import params options = options if options else "" pid_dir = format("{hadoop_pid_dir_prefix}/{user}") pid_file = format("{pid_dir}/hadoop-{user}-{name}.pid") hadoop_env_exports = { 'HADOOP_LIBEXEC_DIR': params.hadoop_libexec_dir } log_dir = format("{hdfs_log_dir_prefix}/{user}") # NFS GATEWAY is always started by root using jsvc due to rpcbind bugs # on Linux such as CentOS6.2. https://bugzilla.redhat.com/show_bug.cgi?id=731542 if name == "nfs3" : pid_file = format("{pid_dir}/hadoop_privileged_nfs3.pid") custom_export = { 'HADOOP_PRIVILEGED_NFS_USER': params.hdfs_user, 'HADOOP_PRIVILEGED_NFS_PID_DIR': pid_dir, 'HADOOP_PRIVILEGED_NFS_LOG_DIR': log_dir } hadoop_env_exports.update(custom_export) process_id_exists_command = as_sudo(["test", "-f", pid_file]) + " && " + as_sudo(["pgrep", "-F", pid_file]) # on STOP directories shouldn't be created # since during stop still old dirs are used (which were created during previous start) if action != "stop": if name == "nfs3": Directory(params.hadoop_pid_dir_prefix, mode=0755, owner=params.root_user, group=params.root_group ) else: Directory(params.hadoop_pid_dir_prefix, mode=0755, owner=params.hdfs_user, group=params.user_group ) if create_pid_dir: Directory(pid_dir, owner=user, recursive=True) if create_log_dir: if name == "nfs3": Directory(log_dir, mode=0775, owner=params.root_user, group=params.user_group) else: Directory(log_dir, owner=user, recursive=True) if params.security_enabled and name == "datanode": ## The directory where pid files are stored in the secure data environment. hadoop_secure_dn_pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}") hadoop_secure_dn_pid_file = format("{hadoop_secure_dn_pid_dir}/hadoop_secure_dn.pid") # At Champlain stack and further, we may start datanode as a non-root even in secure cluster if not (params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0) or params.secure_dn_ports_are_in_use: user = "******" pid_file = format( "{hadoop_pid_dir_prefix}/{hdfs_user}/hadoop-{hdfs_user}-{name}.pid") if action == 'stop' and (params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0) and \ os.path.isfile(hadoop_secure_dn_pid_file): # We need special handling for this case to handle the situation # when we configure non-root secure DN and then restart it # to handle new configs. Otherwise we will not be able to stop # a running instance user = "******" try: check_process_status(hadoop_secure_dn_pid_file) custom_export = { 'HADOOP_SECURE_DN_USER': params.hdfs_user } hadoop_env_exports.update(custom_export) except ComponentIsNotRunning: pass hadoop_daemon = format("{hadoop_bin}/hadoop-daemon.sh") if user == "root": cmd = [hadoop_daemon, "--config", params.hadoop_conf_dir, action, name] if options: cmd += [options, ] daemon_cmd = as_sudo(cmd) else: cmd = format("{ulimit_cmd} {hadoop_daemon} --config {hadoop_conf_dir} {action} {name}") if options: cmd += " " + options daemon_cmd = as_user(cmd, user) if action == "start": # remove pid file from dead process File(pid_file, action="delete", not_if=process_id_exists_command) Execute(daemon_cmd, not_if=process_id_exists_command, environment=hadoop_env_exports) elif action == "stop": Execute(daemon_cmd, only_if=process_id_exists_command, environment=hadoop_env_exports) File(pid_file, action="delete")
def convert_conf_directories_to_symlinks(package, version, dirs, skip_existing_links=True, link_to="current"): """ Assumes HDP 2.3+, moves around directories and creates the conf symlink for the given package. If the package does not exist, then no work is performed. - Creates a /etc/<component>/conf.backup directory - Copies all configs from /etc/<component>/conf to conf.backup - Removes /etc/<component>/conf - Creates /etc/<component>/<version>/0 - Creates /usr/hdp/current/<component>-client/conf -> /etc/<component>/<version>/0 - Links /etc/<component>/conf to <something> -- /etc/<component>/conf -> /usr/hdp/current/[component]-client/conf -- /etc/<component>/conf -> /etc/<component>/conf.backup :param package: the package to create symlinks for (zookeeper, falcon, etc) :param version: the version number to use with conf-select (2.3.0.0-1234) :param dirs: the directories associated with the package (from PACKAGE_DIRS) :param skip_existing_links: True to not do any work if already a symlink :param link_to: link to "current" or "backup" """ bad_dirs = [] for dir_def in dirs: if not os.path.exists(dir_def['conf_dir']): bad_dirs.append(dir_def['conf_dir']) if len(bad_dirs) > 0: Logger.info("Skipping {0} as it does not exist.".format(",".join(bad_dirs))) return # existing links should be skipped since we assume there's no work to do if skip_existing_links: bad_dirs = [] for dir_def in dirs: # check if conf is a link already old_conf = dir_def['conf_dir'] if os.path.islink(old_conf): Logger.info("{0} is already link to {1}".format(old_conf, os.path.realpath(old_conf))) bad_dirs.append(old_conf) if len(bad_dirs) > 0: return # make backup dir and copy everything in case configure() was called after install() backup_dir = None for dir_def in dirs: old_conf = dir_def['conf_dir'] old_parent = os.path.abspath(os.path.join(old_conf, os.pardir)) backup_dir = os.path.join(old_parent, "conf.backup") Logger.info("Backing up {0} to {1} if destination doesn't exist already.".format(old_conf, backup_dir)) Execute(("cp", "-R", "-p", old_conf, backup_dir), not_if = format("test -e {backup_dir}"), sudo = True) # we're already in the HDP stack # Create the versioned /etc/[component]/[version]/0 folder. # The component must be installed on the host. versioned_confs = create("HDP", package, version, dry_run = True) Logger.info("Package {0} will have new conf directories: {1}".format(package, ", ".join(versioned_confs))) need_dirs = [] for d in versioned_confs: if not os.path.exists(d): need_dirs.append(d) if len(need_dirs) > 0: create("HDP", package, version) # find the matching definition and back it up (not the most efficient way) ONLY if there is more than one directory if len(dirs) > 1: for need_dir in need_dirs: for dir_def in dirs: if 'prefix' in dir_def and need_dir.startswith(dir_def['prefix']): old_conf = dir_def['conf_dir'] versioned_conf = need_dir Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False), only_if = format("ls -d {old_conf}/*")) elif 1 == len(dirs) and 1 == len(need_dirs): old_conf = dirs[0]['conf_dir'] versioned_conf = need_dirs[0] Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False), only_if = format("ls -d {old_conf}/*")) # /usr/hdp/current/[component] is already set to to the correct version, e.g., /usr/hdp/[version]/[component] link_component_conf_to_versioned_config(package, version) # Symlink /etc/[component]/conf to /etc/[component]/conf.backup try: # No more references to /etc/[component]/conf for dir_def in dirs: # E.g., /etc/[component]/conf new_symlink = dir_def['conf_dir'] # Remove new_symlink to pave the way, but only if it's a directory if not os.path.islink(new_symlink): Directory(new_symlink, action="delete") if link_to in ["current", "backup"]: # link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf if link_to == "backup": Link(new_symlink, to = backup_dir) else: Link(new_symlink, to = dir_def['current_dir']) else: Logger.error("Unsupported 'link_to' argument. Could not link package {0}".format(package)) except Exception, e: Logger.warning("Could not change symlink for package {0} to point to {1} directory. Error: {2}".format(package, link_to, e))
def service(componentName, action='start', serviceName='yarn'): import params if serviceName == 'mapreduce' and componentName == 'historyserver': delete_pid_file = True daemon = format("{mapred_bin}/mr-jobhistory-daemon.sh") pid_file = format( "{mapred_pid_dir}/mapred-{mapred_user}-{componentName}.pid") usr = params.mapred_user log_dir = params.mapred_log_dir else: # !!! yarn-daemon.sh deletes the PID for us; if we remove it the script # may not work correctly when stopping the service delete_pid_file = False daemon = format("{yarn_bin}/yarn-daemon.sh") pid_file = format( "{yarn_pid_dir}/yarn-{yarn_user}-{componentName}.pid") usr = params.yarn_user log_dir = params.yarn_log_dir cmd = format( "export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {hadoop_conf_dir}" ) if action == 'start': daemon_cmd = format("{ulimit_cmd} {cmd} start {componentName}") check_process = as_sudo(["test", "-f", pid_file]) + " && " + as_sudo( ["pgrep", "-F", pid_file]) # Remove the pid file if its corresponding process is not running. File(pid_file, action="delete", not_if=check_process) if componentName == 'timelineserver' and serviceName == 'yarn': File(params.ats_leveldb_lock_file, action="delete", only_if=format("ls {params.ats_leveldb_lock_file}"), not_if=check_process, ignore_failures=True) try: # Attempt to start the process. Internally, this is skipped if the process is already running. Execute(daemon_cmd, user=usr, not_if=check_process) # Ensure that the process with the expected PID exists. Execute( check_process, not_if=check_process, tries=5, try_sleep=1, ) except: show_logs(log_dir, usr) raise elif action == 'stop': daemon_cmd = format("{cmd} stop {componentName}") try: Execute(daemon_cmd, user=usr) except: show_logs(log_dir, usr) raise # !!! yarn-daemon doesn't need us to delete PIDs if delete_pid_file is True: File(pid_file, action="delete") elif action == 'refreshQueues': rm_kinit_cmd = params.rm_kinit_cmd refresh_cmd = format( "{rm_kinit_cmd} export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {yarn_container_bin}/yarn rmadmin -refreshQueues" ) Execute(refresh_cmd, user=usr)
def convert_conf_directories_to_symlinks(package, version, dirs, skip_existing_links=True, link_to=DIRECTORY_TYPE_CURRENT): """ Assumes HDP 2.3+, moves around directories and creates the conf symlink for the given package. If the package does not exist, then no work is performed. - Creates a /etc/<component>/conf.backup directory - Copies all configs from /etc/<component>/conf to conf.backup - Removes /etc/<component>/conf - Creates /etc/<component>/<version>/0 via <conf-selector-tool> - <stack-root>/current/<component>-client/conf -> /etc/<component>/<version>/0 via <conf-selector-tool> - Links /etc/<component>/conf to <something> depending on function paramter -- /etc/<component>/conf -> <stack-root>/current/[component]-client/conf (usually) -- /etc/<component>/conf -> /etc/<component>/conf.backup (only when supporting < HDP 2.3) :param package: the package to create symlinks for (zookeeper, falcon, etc) :param version: the version number to use with <conf-selector-tool> (2.3.0.0-1234) :param dirs: the directories associated with the package (from get_package_dirs()) :param skip_existing_links: True to not do any work if already a symlink :param link_to: link to "current" or "backup" """ # lack of enums makes this possible - we need to know what to link to if link_to not in [DIRECTORY_TYPE_CURRENT, DIRECTORY_TYPE_BACKUP]: raise Fail( "Unsupported 'link_to' argument. Could not link package {0}". format(package)) stack_name = Script.get_stack_name() bad_dirs = [] for dir_def in dirs: if not os.path.exists(dir_def['conf_dir']): bad_dirs.append(dir_def['conf_dir']) if len(bad_dirs) > 0: Logger.info("Skipping {0} as it does not exist.".format( ",".join(bad_dirs))) return # existing links should be skipped since we assume there's no work to do # they should be checked against the correct target though if skip_existing_links: bad_dirs = [] for dir_def in dirs: # check if conf is a link already old_conf = dir_def['conf_dir'] if os.path.islink(old_conf): # it's already a link; make sure it's a link to where we want it if link_to == DIRECTORY_TYPE_BACKUP: target_conf_dir = _get_backup_conf_directory(old_conf) else: target_conf_dir = dir_def['current_dir'] # the link isn't to the right spot; re-link it if os.readlink(old_conf) != target_conf_dir: Logger.info("Re-linking symlink {0} to {1}".format( old_conf, target_conf_dir)) Link(old_conf, action="delete") Link(old_conf, to=target_conf_dir) else: Logger.info("{0} is already linked to {1}".format( old_conf, os.path.realpath(old_conf))) bad_dirs.append(old_conf) if len(bad_dirs) > 0: return # make backup dir and copy everything in case configure() was called after install() for dir_def in dirs: old_conf = dir_def['conf_dir'] backup_dir = _get_backup_conf_directory(old_conf) Logger.info( "Backing up {0} to {1} if destination doesn't exist already.". format(old_conf, backup_dir)) Execute(("cp", "-R", "-p", old_conf, backup_dir), not_if=format("test -e {backup_dir}"), sudo=True) # we're already in the HDP stack # Create the versioned /etc/[component]/[version]/0 folder. # The component must be installed on the host. versioned_confs = create(stack_name, package, version, dry_run=True) Logger.info("Package {0} will have new conf directories: {1}".format( package, ", ".join(versioned_confs))) need_dirs = [] for d in versioned_confs: if not os.path.exists(d): need_dirs.append(d) if len(need_dirs) > 0: create(stack_name, package, version) # find the matching definition and back it up (not the most efficient way) ONLY if there is more than one directory if len(dirs) > 1: for need_dir in need_dirs: for dir_def in dirs: if 'prefix' in dir_def and need_dir.startswith( dir_def['prefix']): old_conf = dir_def['conf_dir'] versioned_conf = need_dir Execute(as_sudo([ "cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf ], auto_escape=False), only_if=format("ls -d {old_conf}/*")) elif 1 == len(dirs) and 1 == len(need_dirs): old_conf = dirs[0]['conf_dir'] versioned_conf = need_dirs[0] Execute(as_sudo([ "cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf ], auto_escape=False), only_if=format("ls -d {old_conf}/*")) # <stack-root>/current/[component] is already set to to the correct version, e.g., <stack-root>/[version]/[component] select(stack_name, package, version, ignore_errors=True) # Symlink /etc/[component]/conf to /etc/[component]/conf.backup try: # No more references to /etc/[component]/conf for dir_def in dirs: # E.g., /etc/[component]/conf new_symlink = dir_def['conf_dir'] # Delete the existing directory/link so that linking will work if not os.path.islink(new_symlink): Directory(new_symlink, action="delete") else: Link(new_symlink, action="delete") old_conf = dir_def['conf_dir'] backup_dir = _get_backup_conf_directory(old_conf) # link /etc/[component]/conf -> /etc/[component]/conf.backup # or # link /etc/[component]/conf -> <stack-root>/current/[component]-client/conf if link_to == DIRECTORY_TYPE_BACKUP: Link(new_symlink, to=backup_dir) else: Link(new_symlink, to=dir_def['current_dir']) #HACK if package in [ "atlas", ]: Logger.info( "Seeding the new conf symlink {0} from the old backup directory {1} in case any " "unmanaged artifacts are needed.".format( new_symlink, backup_dir)) # If /etc/[component]/conf.backup exists, then copy any artifacts not managed by Ambari to the new symlink target # Be careful not to clobber any existing files. Execute(as_sudo([ "cp", "-R", "--no-clobber", os.path.join(backup_dir, "*"), new_symlink ], auto_escape=False), only_if=format("test -e {new_symlink}")) except Exception, e: Logger.warning( "Could not change symlink for package {0} to point to {1} directory. Error: {2}" .format(package, link_to, e))
def _link_configs(package, version, dirs): """ Link a specific package's configuration directory """ bad_dirs = [] for dir_def in dirs: if not os.path.exists(dir_def['conf_dir']): bad_dirs.append(dir_def['conf_dir']) if len(bad_dirs) > 0: Logger.debug("Skipping {0} as it does not exist.".format(",".join(bad_dirs))) return bad_dirs = [] for dir_def in dirs: # check if conf is a link already old_conf = dir_def['conf_dir'] if os.path.islink(old_conf): Logger.debug("{0} is a link to {1}".format(old_conf, os.path.realpath(old_conf))) bad_dirs.append(old_conf) if len(bad_dirs) > 0: return # make backup dir and copy everything in case configure() was called after install() for dir_def in dirs: old_conf = dir_def['conf_dir'] old_parent = os.path.abspath(os.path.join(old_conf, os.pardir)) old_conf_copy = os.path.join(old_parent, "conf.install") Execute(("cp", "-R", "-p", old_conf, old_conf_copy), not_if = format("test -e {old_conf_copy}"), sudo = True) # we're already in the HDP stack versioned_confs = conf_select.create("HDP", package, version, dry_run = True) Logger.info("New conf directories: {0}".format(", ".join(versioned_confs))) need_dirs = [] for d in versioned_confs: if not os.path.exists(d): need_dirs.append(d) if len(need_dirs) > 0: conf_select.create("HDP", package, version) # find the matching definition and back it up (not the most efficient way) ONLY if there is more than one directory if len(dirs) > 1: for need_dir in need_dirs: for dir_def in dirs: if 'prefix' in dir_def and need_dir.startswith(dir_def['prefix']): old_conf = dir_def['conf_dir'] versioned_conf = need_dir Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False), only_if = format("ls {old_conf}/*")) elif 1 == len(dirs) and 1 == len(need_dirs): old_conf = dirs[0]['conf_dir'] versioned_conf = need_dirs[0] Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False), only_if = format("ls {old_conf}/*")) # make /usr/hdp/[version]/[component]/conf point to the versioned config. # /usr/hdp/current is already set try: conf_select.select("HDP", package, version) # no more references to /etc/[component]/conf for dir_def in dirs: Directory(dir_def['conf_dir'], action="delete") # link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf Link(dir_def['conf_dir'], to = dir_def['current_dir']) except Exception, e: Logger.warning("Could not select the directory: {0}".format(e.message))
def setup_kms_db(): import params if params.has_ranger_admin: password_validation(params.kms_master_key_password, 'KMS master key') File(params.downloaded_custom_connector, content = DownloadSource(params.driver_curl_source), mode = 0644 ) Directory(params.java_share_dir, mode=0755, recursive=True, cd_access="a" ) if params.db_flavor.lower() != 'sqla': Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target), path=["/bin", "/usr/bin/"], sudo=True) File(params.driver_curl_target, mode=0644) Directory(os.path.join(params.kms_home, 'ews', 'lib'), mode=0755 ) if params.db_flavor.lower() == 'sqla': Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True) Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(params.kms_home, 'ews', 'webapp', 'lib')), path=["/bin", "/usr/bin/"], sudo=True) Directory(params.jdbc_libs_dir, cd_access="a", recursive=True) Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False), path=["/bin", "/usr/bin/"]) else: Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(params.kms_home, 'ews', 'webapp', 'lib')), path=["/bin", "/usr/bin/"], sudo=True) File(os.path.join(params.kms_home, 'ews', 'webapp', 'lib', params.jdbc_jar_name), mode=0644) ModifyPropertiesFile(format("/usr/hdp/current/ranger-kms/install.properties"), properties = params.config['configurations']['kms-properties'], owner = params.kms_user ) if params.db_flavor.lower() == 'sqla': ModifyPropertiesFile(format("{kms_home}/install.properties"), properties = {'SQL_CONNECTOR_JAR': format('{kms_home}/ews/webapp/lib/{jdbc_jar_name}')}, owner = params.kms_user, ) env_dict = {'RANGER_KMS_HOME':params.kms_home, 'JAVA_HOME': params.java_home} if params.db_flavor.lower() == 'sqla': env_dict = {'RANGER_KMS_HOME':params.kms_home, 'JAVA_HOME': params.java_home, 'LD_LIBRARY_PATH':params.ld_library_path} dba_setup = format('python {kms_home}/dba_script.py -q') db_setup = format('python {kms_home}/db_setup.py') if params.create_db_user: Logger.info('Setting up Ranger KMS DB and DB User') Execute(dba_setup, environment=env_dict, logoutput=True, user=params.kms_user, tries=5, try_sleep=10) else: Logger.info('Separate DBA property not set. Assuming Ranger KMS DB and DB User exists!') Execute(db_setup, environment=env_dict, logoutput=True, user=params.kms_user, tries=5, try_sleep=10)
def convert_conf_directories_to_symlinks(package, version, dirs, skip_existing_links=True, link_to=DIRECTORY_TYPE_CURRENT): """ Assumes HDP 2.3+, moves around directories and creates the conf symlink for the given package. If the package does not exist, then no work is performed. - Creates a /etc/<component>/conf.backup directory - Copies all configs from /etc/<component>/conf to conf.backup - Removes /etc/<component>/conf - Creates /etc/<component>/<version>/0 via <conf-selector-tool> - <stack-root>/current/<component>-client/conf -> /etc/<component>/<version>/0 via <conf-selector-tool> - Links /etc/<component>/conf to <something> depending on function paramter -- /etc/<component>/conf -> <stack-root>/current/[component]-client/conf (usually) -- /etc/<component>/conf -> /etc/<component>/conf.backup (only when supporting < HDP 2.3) :param package: the package to create symlinks for (zookeeper, falcon, etc) :param version: the version number to use with <conf-selector-tool> (2.3.0.0-1234) :param dirs: the directories associated with the package (from get_package_dirs()) :param skip_existing_links: True to not do any work if already a symlink :param link_to: link to "current" or "backup" """ # lack of enums makes this possible - we need to know what to link to if link_to not in [DIRECTORY_TYPE_CURRENT, DIRECTORY_TYPE_BACKUP]: raise Fail("Unsupported 'link_to' argument. Could not link package {0}".format(package)) stack_name = Script.get_stack_name() bad_dirs = [] for dir_def in dirs: if not os.path.exists(dir_def['conf_dir']): bad_dirs.append(dir_def['conf_dir']) if len(bad_dirs) > 0: Logger.info("Skipping {0} as it does not exist.".format(",".join(bad_dirs))) return # existing links should be skipped since we assume there's no work to do # they should be checked against the correct target though if skip_existing_links: bad_dirs = [] for dir_def in dirs: # check if conf is a link already old_conf = dir_def['conf_dir'] if os.path.islink(old_conf): # it's already a link; make sure it's a link to where we want it if link_to == DIRECTORY_TYPE_BACKUP: target_conf_dir = _get_backup_conf_directory(old_conf) else: target_conf_dir = dir_def['current_dir'] # the link isn't to the right spot; re-link it if os.readlink(old_conf) != target_conf_dir: Logger.info("Re-linking symlink {0} to {1}".format(old_conf, target_conf_dir)) Link(old_conf, action = "delete") Link(old_conf, to = target_conf_dir) else: Logger.info("{0} is already linked to {1}".format(old_conf, os.path.realpath(old_conf))) bad_dirs.append(old_conf) if len(bad_dirs) > 0: return # make backup dir and copy everything in case configure() was called after install() for dir_def in dirs: old_conf = dir_def['conf_dir'] backup_dir = _get_backup_conf_directory(old_conf) Logger.info("Backing up {0} to {1} if destination doesn't exist already.".format(old_conf, backup_dir)) Execute(("cp", "-R", "-p", old_conf, backup_dir), not_if = format("test -e {backup_dir}"), sudo = True) # we're already in the HDP stack # Create the versioned /etc/[component]/[version]/0 folder. # The component must be installed on the host. versioned_confs = create(stack_name, package, version, dry_run = True) Logger.info("Package {0} will have new conf directories: {1}".format(package, ", ".join(versioned_confs))) need_dirs = [] for d in versioned_confs: if not os.path.exists(d): need_dirs.append(d) if len(need_dirs) > 0: create(stack_name, package, version) # find the matching definition and back it up (not the most efficient way) ONLY if there is more than one directory if len(dirs) > 1: for need_dir in need_dirs: for dir_def in dirs: if 'prefix' in dir_def and need_dir.startswith(dir_def['prefix']): old_conf = dir_def['conf_dir'] versioned_conf = need_dir Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False), only_if = format("ls -d {old_conf}/*")) elif 1 == len(dirs) and 1 == len(need_dirs): old_conf = dirs[0]['conf_dir'] versioned_conf = need_dirs[0] Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False), only_if = format("ls -d {old_conf}/*")) # <stack-root>/current/[component] is already set to to the correct version, e.g., <stack-root>/[version]/[component] select(stack_name, package, version, ignore_errors = True) # Symlink /etc/[component]/conf to /etc/[component]/conf.backup try: # No more references to /etc/[component]/conf for dir_def in dirs: # E.g., /etc/[component]/conf new_symlink = dir_def['conf_dir'] # Delete the existing directory/link so that linking will work if not os.path.islink(new_symlink): Directory(new_symlink, action = "delete") else: Link(new_symlink, action = "delete") # link /etc/[component]/conf -> /etc/[component]/conf.backup # or # link /etc/[component]/conf -> <stack-root>/current/[component]-client/conf if link_to == DIRECTORY_TYPE_BACKUP: old_conf = dir_def['conf_dir'] backup_dir = _get_backup_conf_directory(old_conf) Link(new_symlink, to = backup_dir) else: Link(new_symlink, to = dir_def['current_dir']) except Exception, e: Logger.warning("Could not change symlink for package {0} to point to {1} directory. Error: {2}".format(package, link_to, e))
def convert_conf_directories_to_symlinks(package, version, dirs, skip_existing_links=True, link_to="current"): """ Assumes HDP 2.3+, moves around directories and creates the conf symlink for the given package. If the package does not exist, then no work is performed. - Creates a /etc/<component>/conf.backup directory - Copies all configs from /etc/<component>/conf to conf.backup - Removes /etc/<component>/conf - Creates /etc/<component>/<version>/0 - Creates /usr/hdp/current/<component>-client/conf -> /etc/<component>/<version>/0 - Links /etc/<component>/conf to <something> -- /etc/<component>/conf -> /usr/hdp/current/[component]-client/conf -- /etc/<component>/conf -> /etc/<component>/conf.backup :param package: the package to create symlinks for (zookeeper, falcon, etc) :param version: the version number to use with conf-select (2.3.0.0-1234) :param dirs: the directories associated with the package (from PACKAGE_DIRS) :param skip_existing_links: True to not do any work if already a symlink :param link_to: link to "current" or "backup" """ bad_dirs = [] for dir_def in dirs: if not os.path.exists(dir_def['conf_dir']): bad_dirs.append(dir_def['conf_dir']) if len(bad_dirs) > 0: Logger.info("Skipping {0} as it does not exist.".format( ",".join(bad_dirs))) return # existing links should be skipped since we assume there's no work to do if skip_existing_links: bad_dirs = [] for dir_def in dirs: # check if conf is a link already old_conf = dir_def['conf_dir'] if os.path.islink(old_conf): Logger.info("{0} is already linked to {1}".format( old_conf, os.path.realpath(old_conf))) bad_dirs.append(old_conf) if len(bad_dirs) > 0: return # make backup dir and copy everything in case configure() was called after install() backup_dir = None for dir_def in dirs: old_conf = dir_def['conf_dir'] old_parent = os.path.abspath(os.path.join(old_conf, os.pardir)) backup_dir = os.path.join(old_parent, "conf.backup") Logger.info( "Backing up {0} to {1} if destination doesn't exist already.". format(old_conf, backup_dir)) Execute(("cp", "-R", "-p", old_conf, backup_dir), not_if=format("test -e {backup_dir}"), sudo=True) # we're already in the HDP stack # Create the versioned /etc/[component]/[version]/0 folder. # The component must be installed on the host. versioned_confs = create("HDP", package, version, dry_run=True) Logger.info("Package {0} will have new conf directories: {1}".format( package, ", ".join(versioned_confs))) need_dirs = [] for d in versioned_confs: if not os.path.exists(d): need_dirs.append(d) if len(need_dirs) > 0: create("HDP", package, version) # find the matching definition and back it up (not the most efficient way) ONLY if there is more than one directory if len(dirs) > 1: for need_dir in need_dirs: for dir_def in dirs: if 'prefix' in dir_def and need_dir.startswith( dir_def['prefix']): old_conf = dir_def['conf_dir'] versioned_conf = need_dir Execute(as_sudo([ "cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf ], auto_escape=False), only_if=format("ls -d {old_conf}/*")) elif 1 == len(dirs) and 1 == len(need_dirs): old_conf = dirs[0]['conf_dir'] versioned_conf = need_dirs[0] Execute(as_sudo([ "cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf ], auto_escape=False), only_if=format("ls -d {old_conf}/*")) # /usr/hdp/current/[component] is already set to to the correct version, e.g., /usr/hdp/[version]/[component] link_component_conf_to_versioned_config(package, version) # Symlink /etc/[component]/conf to /etc/[component]/conf.backup try: # No more references to /etc/[component]/conf for dir_def in dirs: # E.g., /etc/[component]/conf new_symlink = dir_def['conf_dir'] # Remove new_symlink to pave the way, but only if it's a directory if not os.path.islink(new_symlink): Directory(new_symlink, action="delete") if link_to in ["current", "backup"]: # link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf if link_to == "backup": Link(new_symlink, to=backup_dir) else: Link(new_symlink, to=dir_def['current_dir']) else: Logger.error( "Unsupported 'link_to' argument. Could not link package {0}" .format(package)) except Exception, e: Logger.warning( "Could not change symlink for package {0} to point to {1} directory. Error: {2}" .format(package, link_to, e))
def service(componentName, action='start', serviceName='yarn'): import params if serviceName == 'mapreduce' and componentName == 'historyserver': if not params.hdfs_tmp_dir or params.hdfs_tmp_dir == None or params.hdfs_tmp_dir.lower() == 'null': Logger.error("WARNING: HDFS tmp dir property (hdfs_tmp_dir) is empty or invalid. Ambari will change permissions for the folder on regular basis.") delete_pid_file = True daemon = format("{mapred_bin}/mr-jobhistory-daemon.sh") pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-{componentName}.pid") usr = params.mapred_user log_dir = params.mapred_log_dir else: # !!! yarn-daemon.sh deletes the PID for us; if we remove it the script # may not work correctly when stopping the service delete_pid_file = False daemon = format("{yarn_bin}/yarn-daemon.sh") pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-{componentName}.pid") usr = params.yarn_user log_dir = params.yarn_log_dir cmd = format("export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {hadoop_conf_dir}") if action == 'start': # Check ambari-metrics-hadoop-sink version is less than 2.7.0.0 check_installed_metrics_hadoop_sink_version() daemon_cmd = format("{ulimit_cmd} {cmd} start {componentName}") check_process = as_sudo(["test", "-f", pid_file]) + " && " + as_sudo(["pgrep", "-F", pid_file]) # Remove the pid file if its corresponding process is not running. File(pid_file, action = "delete", not_if = check_process) if componentName == 'timelineserver' and serviceName == 'yarn': File(params.ats_leveldb_lock_file, action = "delete", only_if = format("ls {params.ats_leveldb_lock_file}"), not_if = check_process, ignore_failures = True ) try: # Attempt to start the process. Internally, this is skipped if the process is already running. Execute(daemon_cmd, user = usr, not_if = check_process) # Ensure that the process with the expected PID exists. Execute(check_process, not_if = check_process, tries=5, try_sleep=1, ) except: show_logs(log_dir, usr) raise elif action == 'stop': daemon_cmd = format("{cmd} stop {componentName}") try: Execute(daemon_cmd, user=usr) except: show_logs(log_dir, usr) raise # !!! yarn-daemon doesn't need us to delete PIDs if delete_pid_file is True: File(pid_file, action="delete") elif action == 'refreshQueues': rm_kinit_cmd = params.rm_kinit_cmd refresh_cmd = format("{rm_kinit_cmd} export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {yarn_container_bin}/yarn rmadmin -refreshQueues") Execute(refresh_cmd, user = usr, timeout = 20, # when Yarn is not started command hangs forever and should be killed tries = 5, try_sleep = 5, timeout_kill_strategy = TerminateStrategy.KILL_PROCESS_GROUP, # the process cannot be simply killed by 'kill -15', so kill pg group instread. )
def setup_ranger_db(rolling_upgrade=False): import params File(params.downloaded_custom_connector, content=DownloadSource(params.driver_curl_source), mode=0644) Directory(params.java_share_dir, mode=0755, recursive=True, cd_access="a") if params.db_flavor.lower() != 'sqla': Execute( ('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target), path=["/bin", "/usr/bin/"], sudo=True) File(params.driver_curl_target, mode=0644) ranger_home = params.ranger_home if rolling_upgrade: ranger_home = format("/usr/hdp/{version}/ranger-admin") if params.db_flavor.lower() == 'sqla': Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo=True) Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(params.ranger_home, 'ews', 'lib')), path=["/bin", "/usr/bin/"], sudo=True) Directory(params.jdbc_libs_dir, cd_access="a", recursive=True) Execute(as_sudo([ 'yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir ], auto_escape=False), path=["/bin", "/usr/bin/"]) else: Execute( ('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(params.ranger_home, 'ews', 'lib')), path=["/bin", "/usr/bin/"], sudo=True) File(os.path.join(params.ranger_home, 'ews', 'lib', params.jdbc_jar_name), mode=0644) ModifyPropertiesFile( format("{ranger_home}/install.properties"), properties=params.config['configurations']['admin-properties'], owner=params.unix_user, ) if params.db_flavor.lower() == 'sqla': ModifyPropertiesFile( format("{ranger_home}/install.properties"), properties={ 'SQL_CONNECTOR_JAR': format('{ranger_home}/ews/lib/{jdbc_jar_name}') }, owner=params.unix_user, ) env_dict = { 'RANGER_ADMIN_HOME': ranger_home, 'JAVA_HOME': params.java_home } if params.db_flavor.lower() == 'sqla': env_dict = { 'RANGER_ADMIN_HOME': ranger_home, 'JAVA_HOME': params.java_home, 'LD_LIBRARY_PATH': params.ld_lib_path } # User wants us to setup the DB user and DB? if params.create_db_dbuser: Logger.info('Setting up Ranger DB and DB User') dba_setup = format('ambari-python-wrap {ranger_home}/dba_script.py -q') Execute( dba_setup, environment=env_dict, logoutput=True, user=params.unix_user, ) else: Logger.info( 'Separate DBA property not set. Assuming Ranger DB and DB User exists!' ) db_setup = format('ambari-python-wrap {ranger_home}/db_setup.py') Execute( db_setup, environment=env_dict, logoutput=True, user=params.unix_user, )
def setup_ranger_db(rolling_upgrade=False): import params File(params.downloaded_custom_connector, content = DownloadSource(params.driver_curl_source), mode = 0644 ) Directory(params.java_share_dir, mode=0755, recursive=True, cd_access="a" ) if params.db_flavor.lower() != 'sqla': Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target), path=["/bin", "/usr/bin/"], sudo=True) File(params.driver_curl_target, mode=0644) ranger_home = params.ranger_home if rolling_upgrade: ranger_home = format("/usr/hdp/{version}/ranger-admin") if params.db_flavor.lower() == 'sqla': Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True) Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(params.ranger_home, 'ews', 'lib')), path=["/bin", "/usr/bin/"], sudo=True) Directory(params.jdbc_libs_dir, cd_access="a", recursive=True) Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False), path=["/bin", "/usr/bin/"]) else: Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(params.ranger_home, 'ews', 'lib')), path=["/bin", "/usr/bin/"], sudo=True) File(os.path.join(params.ranger_home, 'ews', 'lib',params.jdbc_jar_name), mode=0644) ModifyPropertiesFile(format("{ranger_home}/install.properties"), properties = params.config['configurations']['admin-properties'], owner = params.unix_user, ) if params.db_flavor.lower() == 'sqla': ModifyPropertiesFile(format("{ranger_home}/install.properties"), properties = {'SQL_CONNECTOR_JAR': format('{ranger_home}/ews/lib/{jdbc_jar_name}')}, owner = params.unix_user, ) env_dict = {'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME':params.java_home} if params.db_flavor.lower() == 'sqla': env_dict = {'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME':params.java_home, 'LD_LIBRARY_PATH':params.ld_lib_path} # User wants us to setup the DB user and DB? if params.create_db_dbuser: Logger.info('Setting up Ranger DB and DB User') dba_setup = format('ambari-python-wrap {ranger_home}/dba_script.py -q') Execute(dba_setup, environment=env_dict, logoutput=True, user=params.unix_user, ) else: Logger.info('Separate DBA property not set. Assuming Ranger DB and DB User exists!') db_setup = format('ambari-python-wrap {ranger_home}/db_setup.py') Execute(db_setup, environment=env_dict, logoutput=True, user=params.unix_user, )
def setup_ranger_admin(upgrade_type=None): import params check_db_connnection() if params.driver_curl_source and not params.driver_curl_source.endswith("/None"): if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar): File(params.previous_jdbc_jar, action='delete') File(params.downloaded_custom_connector, content = DownloadSource(params.driver_curl_source), mode = 0644 ) Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target), path=["/bin", "/usr/bin/"], sudo=True) File(params.driver_curl_target, mode=0644) ModifyPropertiesFile(format("{ranger_home}/install.properties"), properties = params.config['configurations']['admin-properties'] ) custom_config = dict() custom_config['unix_user'] = params.unix_user custom_config['unix_group'] = params.unix_group ModifyPropertiesFile(format("{ranger_home}/install.properties"), properties=custom_config ) ModifyPropertiesFile(format("{ranger_home}/install.properties"), properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')} ) ##if db flavor == oracle - set oracle home env variable if params.db_flavor.lower() == 'oracle' and params.oracle_home: env_dict = {'JAVA_HOME': params.java_home, 'ORACLE_HOME':params.oracle_home, 'LD_LIBRARY_PATH':params.oracle_home} else: env_dict = {'JAVA_HOME': params.java_home} setup_sh = format("cd {ranger_home} && ") + as_sudo([format('{ranger_home}/setup.sh')]) Execute(setup_sh, environment=env_dict, logoutput=True, ) ModifyPropertiesFile(format("{ranger_conf}/xa_system.properties"), properties = params.config['configurations']['ranger-site'], ) ModifyPropertiesFile(format("{ranger_conf}/ranger_webserver.properties"), properties = params.config['configurations']['ranger-site'], mode=0744 ) Directory(params.admin_log_dir, owner = params.unix_user, group = params.unix_group )
def setup_kms_db(): import params if params.has_ranger_admin: File(params.downloaded_custom_connector, content = DownloadSource(params.driver_curl_source), mode = 0644 ) Directory(params.java_share_dir, mode=0755, recursive=True, cd_access="a" ) if params.db_flavor.lower() != 'sqla': Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target), path=["/bin", "/usr/bin/"], sudo=True) File(params.driver_curl_target, mode=0644) Directory(os.path.join(params.kms_home, 'ews', 'lib'), mode=0755 ) if params.db_flavor.lower() == 'sqla': Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True) Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(params.kms_home, 'ews', 'webapp', 'lib')), path=["/bin", "/usr/bin/"], sudo=True) Directory(params.jdbc_libs_dir, cd_access="a", recursive=True) Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False), path=["/bin", "/usr/bin/"]) else: Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(params.kms_home, 'ews', 'webapp', 'lib')), path=["/bin", "/usr/bin/"], sudo=True) File(os.path.join(params.kms_home, 'ews', 'webapp', 'lib', params.jdbc_jar_name), mode=0644) ModifyPropertiesFile(format("/usr/hdp/current/ranger-kms/install.properties"), properties = params.config['configurations']['kms-properties'], owner = params.kms_user ) if params.db_flavor.lower() == 'sqla': ModifyPropertiesFile(format("{kms_home}/install.properties"), properties = {'SQL_CONNECTOR_JAR': format('{kms_home}/ews/webapp/lib/{jdbc_jar_name}')}, owner = params.kms_user, ) env_dict = {'RANGER_KMS_HOME':params.kms_home, 'JAVA_HOME': params.java_home} if params.db_flavor.lower() == 'sqla': env_dict = {'RANGER_KMS_HOME':params.kms_home, 'JAVA_HOME': params.java_home, 'LD_LIBRARY_PATH':params.ld_library_path} dba_setup = format('ambari-python-wrap {kms_home}/dba_script.py -q') db_setup = format('ambari-python-wrap {kms_home}/db_setup.py') Execute(dba_setup, environment=env_dict, logoutput=True, user=params.kms_user) Execute(db_setup, environment=env_dict, logoutput=True, user=params.kms_user)
def _link_configs(package, version, dirs): """ Link a specific package's configuration directory """ bad_dirs = [] for dir_def in dirs: if not os.path.exists(dir_def['conf_dir']): bad_dirs.append(dir_def['conf_dir']) if len(bad_dirs) > 0: Logger.debug("Skipping {0} as it does not exist.".format( ",".join(bad_dirs))) return bad_dirs = [] for dir_def in dirs: # check if conf is a link already old_conf = dir_def['conf_dir'] if os.path.islink(old_conf): Logger.debug("{0} is a link to {1}".format( old_conf, os.path.realpath(old_conf))) bad_dirs.append(old_conf) if len(bad_dirs) > 0: return # make backup dir and copy everything in case configure() was called after install() for dir_def in dirs: old_conf = dir_def['conf_dir'] old_parent = os.path.abspath(os.path.join(old_conf, os.pardir)) old_conf_copy = os.path.join(old_parent, "conf.install") Execute(("cp", "-R", "-p", old_conf, old_conf_copy), not_if=format("test -e {old_conf_copy}"), sudo=True) # we're already in the HDP stack versioned_confs = conf_select.create("HDP", package, version, dry_run=True) Logger.info("New conf directories: {0}".format(", ".join(versioned_confs))) need_dirs = [] for d in versioned_confs: if not os.path.exists(d): need_dirs.append(d) if len(need_dirs) > 0: conf_select.create("HDP", package, version) # find the matching definition and back it up (not the most efficient way) ONLY if there is more than one directory if len(dirs) > 1: for need_dir in need_dirs: for dir_def in dirs: if 'prefix' in dir_def and need_dir.startswith( dir_def['prefix']): old_conf = dir_def['conf_dir'] versioned_conf = need_dir Execute(as_sudo([ "cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf ], auto_escape=False), only_if=format("ls {old_conf}/*")) elif 1 == len(dirs) and 1 == len(need_dirs): old_conf = dirs[0]['conf_dir'] versioned_conf = need_dirs[0] Execute(as_sudo([ "cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf ], auto_escape=False), only_if=format("ls {old_conf}/*")) # make /usr/hdp/[version]/[component]/conf point to the versioned config. # /usr/hdp/current is already set try: conf_select.select("HDP", package, version) # no more references to /etc/[component]/conf for dir_def in dirs: Directory(dir_def['conf_dir'], action="delete") # link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf Link(dir_def['conf_dir'], to=dir_def['current_dir']) except Exception, e: Logger.warning("Could not select the directory: {0}".format(e.message))