def link_configs(struct_out_file): """ Use the conf_select module to link configuration directories correctly. """ import params json_version = load_version(struct_out_file) if not json_version: Logger.info( "Could not load 'version' from {0}".format(struct_out_file)) return if not params.sysprep_skip_conf_select or not os.path.exists( params.conf_select_marker_file): # On parallel command execution this should be executed by a single process at a time. with FcntlBasedProcessLock( params.link_configs_lock_file, enabled=params.is_parallel_execution_enabled, skip_fcntl_failures=True): for package_name, directories in conf_select.get_package_dirs( ).iteritems(): conf_select.convert_conf_directories_to_symlinks( package_name, json_version, directories) # create a file to mark that conf-selects were already done with open(params.conf_select_marker_file, "wb") as fp: pass else: Logger.info( format( "Skipping conf-select stage, since cluster-env/sysprep_skip_conf_select is set and mark file {conf_select_marker_file} exists" ))
def _create_config_links_if_necessary(self, stack_id, stack_version): """ Sets up the required structure for /etc/<component>/conf symlinks and /usr/hdp/current configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3 stack_id: stack id, ie HDP-2.3 stack_version: version to set, ie 2.3.0.0-1234 """ if stack_id is None: Logger.info("Cannot create config links when stack_id is not defined") return args = stack_id.upper().split('-') if len(args) != 2: Logger.info("Unrecognized stack id {0}, cannot create config links".format(stack_id)) return if args[0] != "HDP": Logger.info("Unrecognized stack name {0}, cannot create config links".format(args[0])) if compare_versions(format_hdp_stack_version(args[1]), "2.3.0.0") < 0: Logger.info("Configuration symlinks are not needed for {0}, only HDP-2.3+".format(stack_version)) return for package_name, directories in conf_select.PACKAGE_DIRS.iteritems(): # if already on HDP 2.3, then we should skip making conf.backup folders if self.current_hdp_stack_version and compare_versions(self.current_hdp_stack_version, '2.3') >= 0: Logger.info("The current cluster stack of {0} does not require backing up configurations; " "only conf-select versioned config directories will be created.".format(stack_version)) # only link configs for all known packages conf_select.link_component_conf_to_versioned_config(package_name, stack_version) else: # link configs and create conf.backup folders for all known packages conf_select.convert_conf_directories_to_symlinks(package_name, stack_version, directories, skip_existing_links = False, link_to = "backup")
def _create_config_links_if_necessary(self, stack_id, stack_version): """ Sets up the required structure for /etc/<component>/conf symlinks and <stack-root>/current configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3 stack_id: stack id, ie HDP-2.3 stack_version: version to set, ie 2.3.0.0-1234 """ if stack_id is None: Logger.info("Cannot create config links when stack_id is not defined") return args = stack_id.upper().split('-') if len(args) != 2: Logger.info("Unrecognized stack id {0}, cannot create config links".format(stack_id)) return target_stack_version = args[1] if not (target_stack_version and check_stack_feature(StackFeature.CONFIG_VERSIONING, target_stack_version)): Logger.info("Configuration symlinks are not needed for {0}".format(stack_version)) return for package_name, directories in conf_select.get_package_dirs().iteritems(): # if already on HDP 2.3, then we should skip making conf.backup folders if self.current_stack_version_formatted and check_stack_feature(StackFeature.CONFIG_VERSIONING, self.current_stack_version_formatted): conf_selector_name = stack_tools.get_stack_tool_name(stack_tools.CONF_SELECTOR_NAME) Logger.info("The current cluster stack of {0} does not require backing up configurations; " "only {1} versioned config directories will be created.".format(stack_version, conf_selector_name)) # only link configs for all known packages conf_select.select(self.stack_name, package_name, stack_version, ignore_errors = True) else: # link configs and create conf.backup folders for all known packages # this will also call conf-select select conf_select.convert_conf_directories_to_symlinks(package_name, stack_version, directories, skip_existing_links = False, link_to = "backup")
def create_config_version(self, env): import params for package_name, directories in conf_select.get_package_dirs( ).iteritems(): if package_name == 'registry': conf_select.convert_conf_directories_to_symlinks( package_name, params.current_version, directories)
def _fix_default_links_for_current(self): """ If a prior version of Ambari did not correctly reverse the conf symlinks, then they would be put into a bad state when distributing a new stack. For example: /etc/component/conf (directory) <stack-root>/v1/component/conf -> /etc/component/conf When distributing v2, we'd detect the /etc/component/conf problems and would try to adjust it: /etc/component/conf -> <stack-root>/current/component/conf <stack-root>/v2/component/conf -> /etc/component/v2/0 The problem is that v1 never gets changed (since the stack being distributed is v2), and we end up with a circular link: /etc/component/conf -> <stack-root>/current/component/conf <stack-root>/v1/component/conf -> /etc/component/conf :return: None """ Logger.info( "Attempting to fix any configuration symlinks which are not in the correct state" ) from resource_management.libraries.functions import stack_select restricted_packages = conf_select.get_restricted_packages() if 0 == len(restricted_packages): Logger.info( "There are no restricted conf-select packages for this installation" ) else: Logger.info("Restricting conf-select packages to {0}".format( restricted_packages)) for package_name, directories in conf_select.get_package_dirs( ).iteritems(): Logger.info( "Attempting to fix the default conf links for {0}".format( package_name)) Logger.info( "The following directories will be fixed for {0}: {1}".format( package_name, str(directories))) component_name = None for directory_struct in directories: if "component" in directory_struct: component_name = directory_struct["component"] if component_name: stack_version = stack_select.get_stack_version_before_install( component_name) if 0 == len(restricted_packages ) or package_name in restricted_packages: if stack_version: conf_select.convert_conf_directories_to_symlinks( package_name, stack_version, directories) else: Logger.warning( "Unable to fix {0} since there is no known installed version for this component" .format(package_name))
def test_symlink_noop(self): """ Tests that conf-select symlinking does nothing if the directory doesn't exist :return: """ conf_select.convert_conf_directories_to_symlinks("hadoop", "2.3.0.0-1234", conf_select.PACKAGE_DIRS["hadoop"], link_to = conf_select.DIRECTORY_TYPE_BACKUP) self.assertEqual(pprint.pformat(self.env.resource_list), "[]")
def test_symlink_conversion_relinks_wrong_link(self): """ Tests that conf-select symlinking can detect a wrong directory :return: """ conf_select.convert_conf_directories_to_symlinks("hadoop", "2.3.0.0-1234", conf_select.PACKAGE_DIRS["hadoop"]) self.assertEqual(pprint.pformat(self.env.resource_list), "[Link['/etc/hadoop/conf'], Link['/etc/hadoop/conf']]")
def test_symlink_noop(self): """ Tests that conf-select symlinking does nothing if the directory doesn't exist :return: """ packages = conf_select.get_package_dirs() conf_select.convert_conf_directories_to_symlinks( "hadoop", "2.3.0.0-1234", packages["hadoop"]) self.assertEqual(pprint.pformat(self.env.resource_list), "[]")
def test_symlink_conversion_bad_linkto(self): """ Tests that a bad enum throws an exception. :return: """ try: conf_select.convert_conf_directories_to_symlinks("hadoop", "2.3.0.0-1234", conf_select._PACKAGE_DIRS["hadoop"], link_to = "INVALID") raise Exception("Expected failure when supplying a bad enum for link_to") except: pass
def _create_config_links_if_necessary(self, stack_id, stack_version): """ Sets up the required structure for /etc/<component>/conf symlinks and /usr/hdp/current configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3 stack_id: stack id, ie HDP-2.3 stack_version: version to set, ie 2.3.0.0-1234 """ if stack_id is None: Logger.info( "Cannot create config links when stack_id is not defined") return args = stack_id.upper().split('-') if len(args) != 2: Logger.info( "Unrecognized stack id {0}, cannot create config links".format( stack_id)) return if args[0] != "HDP": Logger.info( "Unrecognized stack name {0}, cannot create config links". format(args[0])) if compare_versions(format_hdp_stack_version(args[1]), "2.3.0.0") < 0: Logger.info( "Configuration symlinks are not needed for {0}, only HDP-2.3+". format(stack_version)) return for package_name, directories in conf_select.PACKAGE_DIRS.iteritems(): # if already on HDP 2.3, then we should skip making conf.backup folders if self.current_hdp_stack_version and compare_versions( self.current_hdp_stack_version, '2.3') >= 0: Logger.info( "The current cluster stack of {0} does not require backing up configurations; " "only conf-select versioned config directories will be created." .format(stack_version)) # only link configs for all known packages conf_select.select("HDP", package_name, stack_version, ignore_errors=True) else: # link configs and create conf.backup folders for all known packages # this will also call conf-select select conf_select.convert_conf_directories_to_symlinks( package_name, stack_version, directories, skip_existing_links=False, link_to="backup")
def test_symlink_conversion_to_current(self, islink_mock, path_mock, isdir_mock, shell_call_mock): """ Tests that conf-select creates the correct symlink directories. :return: """ def mock_call(command, **kwargs): """ Instead of shell.call, call a command whose output equals the command. :param command: Command that will be echoed. :return: Returns a tuple of (process output code, stdout, stderr) """ return (0, "/etc/hadoop/conf", None) def path_mock_call(path): if path == "/etc/hadoop/conf": return True if path == "/etc/hadoop/2.3.0.0-1234/0": return True return False def islink_mock_call(path): if path == "/etc/hadoop/conf": return False return False def isdir_mock_call(path): if path == "/etc/hadoop/conf": return True return False packages = conf_select.get_package_dirs() path_mock.side_effect = path_mock_call islink_mock.side_effect = islink_mock_call shell_call_mock.side_effect = mock_call conf_select.convert_conf_directories_to_symlinks( "hadoop", "2.3.0.0-1234", packages["hadoop"]) self.assertEqual( pprint.pformat(self.env.resource_list[0]), "Execute[('cp', '-R', '-p', u'/etc/hadoop/conf', u'/etc/hadoop/conf.backup')]" ) self.assertEqual(pprint.pformat(self.env.resource_list[1]), "Directory['/etc/hadoop/conf']") self.assertEqual(pprint.pformat(self.env.resource_list[2]), "Link['/etc/hadoop/conf']")
def _relink_configurations_with_conf_select(self, stack_id, stack_version): """ Sets up the required structure for /etc/<component>/conf symlinks and <stack-root>/current configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3 stack_id: stack id, ie HDP-2.3 stack_version: version to set, ie 2.3.0.0-1234 """ if stack_id is None: Logger.info( "Cannot create config links when stack_id is not defined") return args = stack_id.upper().split('-') if len(args) != 2: Logger.info( "Unrecognized stack id {0}, cannot create config links".format( stack_id)) return target_stack_version = args[1] if not (target_stack_version and check_stack_feature( StackFeature.CONFIG_VERSIONING, target_stack_version)): Logger.info("Configuration symlinks are not needed for {0}".format( stack_version)) return # After upgrading hdf-select package from HDF-2.X to HDF-3.Y, we need to create this symlink if self.stack_name.upper() == "HDF" \ and not sudo.path_exists("/usr/bin/conf-select") and sudo.path_exists("/usr/bin/hdfconf-select"): Link("/usr/bin/conf-select", to="/usr/bin/hdfconf-select") restricted_packages = conf_select.get_restricted_packages() if 0 == len(restricted_packages): Logger.info( "There are no restricted conf-select packages for this installation" ) else: Logger.info("Restricting conf-select packages to {0}".format( restricted_packages)) for package_name, directories in conf_select.get_package_dirs( ).iteritems(): if 0 == len(restricted_packages ) or package_name in restricted_packages: conf_select.convert_conf_directories_to_symlinks( package_name, stack_version, directories)
def link_configs(struct_out_file): """ Links configs, only on a fresh install of HDP-2.3 and higher """ import params json_version = load_version(struct_out_file) if not json_version: Logger.info("Could not load 'version' from {0}".format(struct_out_file)) return # On parallel command execution this should be executed by a single process at a time. with FcntlBasedProcessLock(params.link_configs_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True): for k, v in conf_select.get_package_dirs().iteritems(): conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def link_configs(struct_out_file): """ Use the conf_select module to link configuration directories correctly. """ import params json_version = load_version(struct_out_file) if not json_version: Logger.info("Could not load 'version' from {0}".format(struct_out_file)) return # On parallel command execution this should be executed by a single process at a time. with FcntlBasedProcessLock(params.link_configs_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True): for package_name, directories in conf_select.get_package_dirs().iteritems(): conf_select.convert_conf_directories_to_symlinks(package_name, json_version, directories)
def link_configs(struct_out_file): """ Links configs, only on a fresh install of HDP-2.3 and higher """ if not Script.is_hdp_stack_greater_or_equal("2.3"): Logger.info("Can only link configs for HDP-2.3 and higher.") return json_version = load_version(struct_out_file) if not json_version: Logger.info("Could not load 'version' from {0}".format(struct_out_file)) return for k, v in conf_select.PACKAGE_DIRS.iteritems(): conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def link_configs(struct_out_file): """ Links configs, only on a fresh install of BigInsights-4.1 and higher """ if not Script.is_stack_greater_or_equal("4.1"): Logger.info("Can only link configs for BigInsights-4.1 and higher.") return json_version = load_version(struct_out_file) if not json_version: Logger.info( "Could not load 'version' from {0}".format(struct_out_file)) return for k, v in conf_select.get_package_dirs().iteritems(): conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def link_configs(struct_out_file): """ Links configs, only on a fresh install of HDP-2.3 and higher """ if not Script.is_hdp_stack_greater_or_equal("2.3"): Logger.info("Can only link configs for HDP-2.3 and higher.") return json_version = load_version(struct_out_file) if not json_version: Logger.info( "Could not load 'version' from {0}".format(struct_out_file)) return for k, v in conf_select.PACKAGE_DIRS.iteritems(): conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def _fix_default_links(self, package_name, component_name): """ If a prior version of Ambari did not correctly reverse the conf symlinks, then they would be put into a bad state when distributing a new stack. For example: /etc/component/conf (directory) <stack-root>/v1/component/conf -> /etc/component/conf When distributing v2, we'd detect the /etc/component/conf problems and would try to adjust it: /etc/component/conf -> <stack-root>/current/component/conf <stack-root>/v2/component/conf -> /etc/component/v2/0 The problem is that v1 never gets changed (since the stack being distributed is v2), and we end up with a circular link: /etc/component/conf -> <stack-root>/current/component/conf <stack-root>/v1/component/conf -> /etc/component/conf :return: None """ from resource_management.libraries.functions import stack_select package_dirs = conf_select.get_package_dirs() if package_name in package_dirs: Logger.info( "Determining if the default conf links for {0} need to be fixed" .format(package_name)) directories = package_dirs[package_name] Logger.info( "The following directories will be checked for {0}: {1}". format(package_name, str(directories))) stack_version = stack_select.get_stack_version_before_install( component_name) if stack_version: conf_select.convert_conf_directories_to_symlinks( package_name, stack_version, directories)
def create_30_config_version(self, env): package_name = 'registry' stack_root = Script.get_stack_root() current_dir = "{0}/current/registry/conf".format(stack_root) directories = [{ "conf_dir": "/etc/registry/conf", "current_dir": current_dir }] stack_version = stack_select.get_stack_version_before_install( package_name) conf_dir = "/etc/registry/conf" if stack_version: try: #Check if broken symbolic links issue exists os.stat(conf_dir) conf_select.convert_conf_directories_to_symlinks( package_name, stack_version, directories) cp_cmd = as_sudo([ "cp", "-a", "-f", "/etc/registry/conf.backup/.", "/etc/registry/conf" ]) Execute(cp_cmd, logoutput=True) except OSError as e: Logger.warning( "Detected broken symlink : {0}. Attempting to repair.". format(str(e))) #removing symlink conf directory sudo.unlink(conf_dir) #make conf dir again sudo.makedirs(conf_dir, 0755) #copy all files for files in glob.glob("/etc/registry/conf.backup/*"): cp_cmd = as_sudo(["cp", "-r", files, conf_dir]) Execute(cp_cmd, logoutput=True) conf_select.convert_conf_directories_to_symlinks( package_name, stack_version, directories)