def link_configs(struct_out_file):
"""
Use the conf_select module to link configuration directories correctly.
"""
import params
json_version = load_version(struct_out_file)
if not json_version:
Logger.info(
"Could not load 'version' from {0}".format(struct_out_file))
return
if not params.sysprep_skip_conf_select or not os.path.exists(
params.conf_select_marker_file):
# On parallel command execution this should be executed by a single process at a time.
with FcntlBasedProcessLock(
params.link_configs_lock_file,
enabled=params.is_parallel_execution_enabled,
skip_fcntl_failures=True):
for package_name, directories in conf_select.get_package_dirs(
).iteritems():
conf_select.convert_conf_directories_to_symlinks(
package_name, json_version, directories)
# create a file to mark that conf-selects were already done
with open(params.conf_select_marker_file, "wb") as fp:
pass
else:
Logger.info(
format(
"Skipping conf-select stage, since cluster-env/sysprep_skip_conf_select is set and mark file {conf_select_marker_file} exists"
))
def _create_config_links_if_necessary(self, stack_id, stack_version):
"""
Sets up the required structure for /etc/<component>/conf symlinks and /usr/hdp/current
configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3
stack_id: stack id, ie HDP-2.3
stack_version: version to set, ie 2.3.0.0-1234
"""
if stack_id is None:
Logger.info("Cannot create config links when stack_id is not defined")
return
args = stack_id.upper().split('-')
if len(args) != 2:
Logger.info("Unrecognized stack id {0}, cannot create config links".format(stack_id))
return
if args[0] != "HDP":
Logger.info("Unrecognized stack name {0}, cannot create config links".format(args[0]))
if compare_versions(format_hdp_stack_version(args[1]), "2.3.0.0") < 0:
Logger.info("Configuration symlinks are not needed for {0}, only HDP-2.3+".format(stack_version))
return
for package_name, directories in conf_select.PACKAGE_DIRS.iteritems():
# if already on HDP 2.3, then we should skip making conf.backup folders
if self.current_hdp_stack_version and compare_versions(self.current_hdp_stack_version, '2.3') >= 0:
Logger.info("The current cluster stack of {0} does not require backing up configurations; "
"only conf-select versioned config directories will be created.".format(stack_version))
# only link configs for all known packages
conf_select.link_component_conf_to_versioned_config(package_name, stack_version)
else:
# link configs and create conf.backup folders for all known packages
conf_select.convert_conf_directories_to_symlinks(package_name, stack_version, directories,
skip_existing_links = False, link_to = "backup")
def _create_config_links_if_necessary(self, stack_id, stack_version):
"""
Sets up the required structure for /etc/<component>/conf symlinks and <stack-root>/current
configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3
stack_id: stack id, ie HDP-2.3
stack_version: version to set, ie 2.3.0.0-1234
"""
if stack_id is None:
Logger.info("Cannot create config links when stack_id is not defined")
return
args = stack_id.upper().split('-')
if len(args) != 2:
Logger.info("Unrecognized stack id {0}, cannot create config links".format(stack_id))
return
target_stack_version = args[1]
if not (target_stack_version and check_stack_feature(StackFeature.CONFIG_VERSIONING, target_stack_version)):
Logger.info("Configuration symlinks are not needed for {0}".format(stack_version))
return
for package_name, directories in conf_select.get_package_dirs().iteritems():
# if already on HDP 2.3, then we should skip making conf.backup folders
if self.current_stack_version_formatted and check_stack_feature(StackFeature.CONFIG_VERSIONING, self.current_stack_version_formatted):
conf_selector_name = stack_tools.get_stack_tool_name(stack_tools.CONF_SELECTOR_NAME)
Logger.info("The current cluster stack of {0} does not require backing up configurations; "
"only {1} versioned config directories will be created.".format(stack_version, conf_selector_name))
# only link configs for all known packages
conf_select.select(self.stack_name, package_name, stack_version, ignore_errors = True)
else:
# link configs and create conf.backup folders for all known packages
# this will also call conf-select select
conf_select.convert_conf_directories_to_symlinks(package_name, stack_version, directories,
skip_existing_links = False, link_to = "backup")
def create_config_version(self, env):
import params
for package_name, directories in conf_select.get_package_dirs(
).iteritems():
if package_name == 'registry':
conf_select.convert_conf_directories_to_symlinks(
package_name, params.current_version, directories)
def _fix_default_links_for_current(self):
"""
If a prior version of Ambari did not correctly reverse the conf symlinks, then they would
be put into a bad state when distributing a new stack. For example:
/etc/component/conf (directory)
<stack-root>/v1/component/conf -> /etc/component/conf
When distributing v2, we'd detect the /etc/component/conf problems and would try to adjust it:
/etc/component/conf -> <stack-root>/current/component/conf
<stack-root>/v2/component/conf -> /etc/component/v2/0
The problem is that v1 never gets changed (since the stack being distributed is v2), and
we end up with a circular link:
/etc/component/conf -> <stack-root>/current/component/conf
<stack-root>/v1/component/conf -> /etc/component/conf
:return: None
"""
Logger.info(
"Attempting to fix any configuration symlinks which are not in the correct state"
)
from resource_management.libraries.functions import stack_select
restricted_packages = conf_select.get_restricted_packages()
if 0 == len(restricted_packages):
Logger.info(
"There are no restricted conf-select packages for this installation"
)
else:
Logger.info("Restricting conf-select packages to {0}".format(
restricted_packages))
for package_name, directories in conf_select.get_package_dirs(
).iteritems():
Logger.info(
"Attempting to fix the default conf links for {0}".format(
package_name))
Logger.info(
"The following directories will be fixed for {0}: {1}".format(
package_name, str(directories)))
component_name = None
for directory_struct in directories:
if "component" in directory_struct:
component_name = directory_struct["component"]
if component_name:
stack_version = stack_select.get_stack_version_before_install(
component_name)
if 0 == len(restricted_packages
) or package_name in restricted_packages:
if stack_version:
conf_select.convert_conf_directories_to_symlinks(
package_name, stack_version, directories)
else:
Logger.warning(
"Unable to fix {0} since there is no known installed version for this component"
.format(package_name))
def test_symlink_noop(self):
"""
Tests that conf-select symlinking does nothing if the directory doesn't exist
:return:
"""
conf_select.convert_conf_directories_to_symlinks("hadoop", "2.3.0.0-1234",
conf_select.PACKAGE_DIRS["hadoop"], link_to = conf_select.DIRECTORY_TYPE_BACKUP)
self.assertEqual(pprint.pformat(self.env.resource_list), "[]")
def test_symlink_conversion_relinks_wrong_link(self):
"""
Tests that conf-select symlinking can detect a wrong directory
:return:
"""
conf_select.convert_conf_directories_to_symlinks("hadoop", "2.3.0.0-1234",
conf_select.PACKAGE_DIRS["hadoop"])
self.assertEqual(pprint.pformat(self.env.resource_list),
"[Link['/etc/hadoop/conf'], Link['/etc/hadoop/conf']]")
def test_symlink_noop(self):
"""
Tests that conf-select symlinking does nothing if the directory doesn't exist
:return:
"""
packages = conf_select.get_package_dirs()
conf_select.convert_conf_directories_to_symlinks(
"hadoop", "2.3.0.0-1234", packages["hadoop"])
self.assertEqual(pprint.pformat(self.env.resource_list), "[]")
def test_symlink_conversion_bad_linkto(self):
"""
Tests that a bad enum throws an exception.
:return:
"""
try:
conf_select.convert_conf_directories_to_symlinks("hadoop", "2.3.0.0-1234",
conf_select._PACKAGE_DIRS["hadoop"], link_to = "INVALID")
raise Exception("Expected failure when supplying a bad enum for link_to")
except:
pass
def _create_config_links_if_necessary(self, stack_id, stack_version):
"""
Sets up the required structure for /etc/<component>/conf symlinks and /usr/hdp/current
configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3
stack_id: stack id, ie HDP-2.3
stack_version: version to set, ie 2.3.0.0-1234
"""
if stack_id is None:
Logger.info(
"Cannot create config links when stack_id is not defined")
return
args = stack_id.upper().split('-')
if len(args) != 2:
Logger.info(
"Unrecognized stack id {0}, cannot create config links".format(
stack_id))
return
if args[0] != "HDP":
Logger.info(
"Unrecognized stack name {0}, cannot create config links".
format(args[0]))
if compare_versions(format_hdp_stack_version(args[1]), "2.3.0.0") < 0:
Logger.info(
"Configuration symlinks are not needed for {0}, only HDP-2.3+".
format(stack_version))
return
for package_name, directories in conf_select.PACKAGE_DIRS.iteritems():
# if already on HDP 2.3, then we should skip making conf.backup folders
if self.current_hdp_stack_version and compare_versions(
self.current_hdp_stack_version, '2.3') >= 0:
Logger.info(
"The current cluster stack of {0} does not require backing up configurations; "
"only conf-select versioned config directories will be created."
.format(stack_version))
# only link configs for all known packages
conf_select.select("HDP",
package_name,
stack_version,
ignore_errors=True)
else:
# link configs and create conf.backup folders for all known packages
# this will also call conf-select select
conf_select.convert_conf_directories_to_symlinks(
package_name,
stack_version,
directories,
skip_existing_links=False,
link_to="backup")
def test_symlink_conversion_to_current(self, islink_mock, path_mock,
isdir_mock, shell_call_mock):
"""
Tests that conf-select creates the correct symlink directories.
:return:
"""
def mock_call(command, **kwargs):
"""
Instead of shell.call, call a command whose output equals the command.
:param command: Command that will be echoed.
:return: Returns a tuple of (process output code, stdout, stderr)
"""
return (0, "/etc/hadoop/conf", None)
def path_mock_call(path):
if path == "/etc/hadoop/conf":
return True
if path == "/etc/hadoop/2.3.0.0-1234/0":
return True
return False
def islink_mock_call(path):
if path == "/etc/hadoop/conf":
return False
return False
def isdir_mock_call(path):
if path == "/etc/hadoop/conf":
return True
return False
packages = conf_select.get_package_dirs()
path_mock.side_effect = path_mock_call
islink_mock.side_effect = islink_mock_call
shell_call_mock.side_effect = mock_call
conf_select.convert_conf_directories_to_symlinks(
"hadoop", "2.3.0.0-1234", packages["hadoop"])
self.assertEqual(
pprint.pformat(self.env.resource_list[0]),
"Execute[('cp', '-R', '-p', u'/etc/hadoop/conf', u'/etc/hadoop/conf.backup')]"
)
self.assertEqual(pprint.pformat(self.env.resource_list[1]),
"Directory['/etc/hadoop/conf']")
self.assertEqual(pprint.pformat(self.env.resource_list[2]),
"Link['/etc/hadoop/conf']")
def _relink_configurations_with_conf_select(self, stack_id, stack_version):
"""
Sets up the required structure for /etc/<component>/conf symlinks and <stack-root>/current
configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3
stack_id: stack id, ie HDP-2.3
stack_version: version to set, ie 2.3.0.0-1234
"""
if stack_id is None:
Logger.info(
"Cannot create config links when stack_id is not defined")
return
args = stack_id.upper().split('-')
if len(args) != 2:
Logger.info(
"Unrecognized stack id {0}, cannot create config links".format(
stack_id))
return
target_stack_version = args[1]
if not (target_stack_version and check_stack_feature(
StackFeature.CONFIG_VERSIONING, target_stack_version)):
Logger.info("Configuration symlinks are not needed for {0}".format(
stack_version))
return
# After upgrading hdf-select package from HDF-2.X to HDF-3.Y, we need to create this symlink
if self.stack_name.upper() == "HDF" \
and not sudo.path_exists("/usr/bin/conf-select") and sudo.path_exists("/usr/bin/hdfconf-select"):
Link("/usr/bin/conf-select", to="/usr/bin/hdfconf-select")
restricted_packages = conf_select.get_restricted_packages()
if 0 == len(restricted_packages):
Logger.info(
"There are no restricted conf-select packages for this installation"
)
else:
Logger.info("Restricting conf-select packages to {0}".format(
restricted_packages))
for package_name, directories in conf_select.get_package_dirs(
).iteritems():
if 0 == len(restricted_packages
) or package_name in restricted_packages:
conf_select.convert_conf_directories_to_symlinks(
package_name, stack_version, directories)
def link_configs(struct_out_file):
"""
Links configs, only on a fresh install of HDP-2.3 and higher
"""
import params
json_version = load_version(struct_out_file)
if not json_version:
Logger.info("Could not load 'version' from {0}".format(struct_out_file))
return
# On parallel command execution this should be executed by a single process at a time.
with FcntlBasedProcessLock(params.link_configs_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True):
for k, v in conf_select.get_package_dirs().iteritems():
conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def link_configs(struct_out_file):
"""
Use the conf_select module to link configuration directories correctly.
"""
import params
json_version = load_version(struct_out_file)
if not json_version:
Logger.info("Could not load 'version' from {0}".format(struct_out_file))
return
# On parallel command execution this should be executed by a single process at a time.
with FcntlBasedProcessLock(params.link_configs_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True):
for package_name, directories in conf_select.get_package_dirs().iteritems():
conf_select.convert_conf_directories_to_symlinks(package_name, json_version, directories)
def link_configs(struct_out_file):
"""
Links configs, only on a fresh install of HDP-2.3 and higher
"""
if not Script.is_hdp_stack_greater_or_equal("2.3"):
Logger.info("Can only link configs for HDP-2.3 and higher.")
return
json_version = load_version(struct_out_file)
if not json_version:
Logger.info("Could not load 'version' from {0}".format(struct_out_file))
return
for k, v in conf_select.PACKAGE_DIRS.iteritems():
conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def link_configs(struct_out_file):
"""
Links configs, only on a fresh install of BigInsights-4.1 and higher
"""
if not Script.is_stack_greater_or_equal("4.1"):
Logger.info("Can only link configs for BigInsights-4.1 and higher.")
return
json_version = load_version(struct_out_file)
if not json_version:
Logger.info(
"Could not load 'version' from {0}".format(struct_out_file))
return
for k, v in conf_select.get_package_dirs().iteritems():
conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def link_configs(struct_out_file):
"""
Links configs, only on a fresh install of HDP-2.3 and higher
"""
if not Script.is_hdp_stack_greater_or_equal("2.3"):
Logger.info("Can only link configs for HDP-2.3 and higher.")
return
json_version = load_version(struct_out_file)
if not json_version:
Logger.info(
"Could not load 'version' from {0}".format(struct_out_file))
return
for k, v in conf_select.PACKAGE_DIRS.iteritems():
conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def _fix_default_links(self, package_name, component_name):
"""
If a prior version of Ambari did not correctly reverse the conf symlinks, then they would
be put into a bad state when distributing a new stack. For example:
/etc/component/conf (directory)
<stack-root>/v1/component/conf -> /etc/component/conf
When distributing v2, we'd detect the /etc/component/conf problems and would try to adjust it:
/etc/component/conf -> <stack-root>/current/component/conf
<stack-root>/v2/component/conf -> /etc/component/v2/0
The problem is that v1 never gets changed (since the stack being distributed is v2), and
we end up with a circular link:
/etc/component/conf -> <stack-root>/current/component/conf
<stack-root>/v1/component/conf -> /etc/component/conf
:return: None
"""
from resource_management.libraries.functions import stack_select
package_dirs = conf_select.get_package_dirs()
if package_name in package_dirs:
Logger.info(
"Determining if the default conf links for {0} need to be fixed"
.format(package_name))
directories = package_dirs[package_name]
Logger.info(
"The following directories will be checked for {0}: {1}".
format(package_name, str(directories)))
stack_version = stack_select.get_stack_version_before_install(
component_name)
if stack_version:
conf_select.convert_conf_directories_to_symlinks(
package_name, stack_version, directories)
def create_30_config_version(self, env):
package_name = 'registry'
stack_root = Script.get_stack_root()
current_dir = "{0}/current/registry/conf".format(stack_root)
directories = [{
"conf_dir": "/etc/registry/conf",
"current_dir": current_dir
}]
stack_version = stack_select.get_stack_version_before_install(
package_name)
conf_dir = "/etc/registry/conf"
if stack_version:
try:
#Check if broken symbolic links issue exists
os.stat(conf_dir)
conf_select.convert_conf_directories_to_symlinks(
package_name, stack_version, directories)
cp_cmd = as_sudo([
"cp", "-a", "-f", "/etc/registry/conf.backup/.",
"/etc/registry/conf"
])
Execute(cp_cmd, logoutput=True)
except OSError as e:
Logger.warning(
"Detected broken symlink : {0}. Attempting to repair.".
format(str(e)))
#removing symlink conf directory
sudo.unlink(conf_dir)
#make conf dir again
sudo.makedirs(conf_dir, 0755)
#copy all files
for files in glob.glob("/etc/registry/conf.backup/*"):
cp_cmd = as_sudo(["cp", "-r", files, conf_dir])
Execute(cp_cmd, logoutput=True)
conf_select.convert_conf_directories_to_symlinks(
package_name, stack_version, directories)
