def setup_stack_symlinks(struct_out_file):
"""
Invokes <stack-selector-tool> set all against a calculated fully-qualified, "normalized" version based on a
stack version, such as "2.3". This should always be called after a component has been
installed to ensure that all HDP pointers are correct. The stack upgrade logic does not
interact with this since it's done via a custom command and will not trigger this hook.
:return:
"""
import params
if params.upgrade_suspended:
Logger.warning("Skipping running stack-selector-tool because there is a suspended upgrade")
return
if params.host_sys_prepped:
Logger.warning("Skipping running stack-selector-tool becase this is a sys_prepped host. This may cause symlink pointers not to be created for HDP componets installed later on top of an already sys_prepped host.")
return
# get the packages which the stack-select tool should be used on
stack_select_packages = stack_select.get_packages(stack_select.PACKAGE_SCOPE_INSTALL)
if stack_select_packages is None:
return
json_version = load_version(struct_out_file)
if not json_version:
Logger.info("There is no advertised version for this component stored in {0}".format(struct_out_file))
return
# On parallel command execution this should be executed by a single process at a time.
with FcntlBasedProcessLock(params.stack_select_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True):
for package in stack_select_packages:
stack_select.select(package, json_version)
def locking_configure(obj, *args, **kw):
# local import to avoid circular dependency (default imports Script)
from resource_management.libraries.functions.default import default
parallel_execution_enabled = int(default("/agentConfigParams/agent/parallel_execution", 0)) == 1
lock = FcntlBasedProcessLock(get_config_lock_file(), skip_fcntl_failures = True, enabled = parallel_execution_enabled)
with lock:
original_configure(obj, *args, **kw)
def link_configs(struct_out_file):
"""
Use the conf_select module to link configuration directories correctly.
"""
import params
json_version = load_version(struct_out_file)
if not json_version:
Logger.info(
"Could not load 'version' from {0}".format(struct_out_file))
return
if not params.sysprep_skip_conf_select or not os.path.exists(
params.conf_select_marker_file):
# On parallel command execution this should be executed by a single process at a time.
with FcntlBasedProcessLock(
params.link_configs_lock_file,
enabled=params.is_parallel_execution_enabled,
skip_fcntl_failures=True):
for package_name, directories in conf_select.get_package_dirs(
).iteritems():
conf_select.convert_conf_directories_to_symlinks(
package_name, json_version, directories)
# create a file to mark that conf-selects were already done
with open(params.conf_select_marker_file, "wb") as fp:
pass
else:
Logger.info(
format(
"Skipping conf-select stage, since cluster-env/sysprep_skip_conf_select is set and mark file {conf_select_marker_file} exists"
))
def dummy_task(index, mutex):
with FcntlBasedProcessLock(lock_file,
skip_fcntl_failures=False):
if (not mutex.acquire(block=False)):
raise Exception(
"ERROR: FcntlBasedProcessLock was acquired by several processes"
)
time.sleep(0.1)
mutex.release()
def link_configs(struct_out_file):
"""
Use the conf_select module to link configuration directories correctly.
"""
import params
json_version = load_version(struct_out_file)
if not json_version:
Logger.info("Could not load 'version' from {0}".format(struct_out_file))
return
# On parallel command execution this should be executed by a single process at a time.
with FcntlBasedProcessLock(params.link_configs_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True):
for package_name, directories in conf_select.get_package_dirs().iteritems():
conf_select.convert_conf_directories_to_symlinks(package_name, json_version, directories)
def link_configs(struct_out_file):
"""
Links configs, only on a fresh install of HDP-2.3 and higher
"""
import params
json_version = load_version(struct_out_file)
if not json_version:
Logger.info("Could not load 'version' from {0}".format(struct_out_file))
return
# On parallel command execution this should be executed by a single process at a time.
with FcntlBasedProcessLock(params.link_configs_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True):
for k, v in conf_select.get_package_dirs().iteritems():
conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def setup_stack_symlinks():
"""
Invokes <stack-selector-tool> set all against a calculated fully-qualified, "normalized" version based on a
stack version, such as "2.3". This should always be called after a component has been
installed to ensure that all HDP pointers are correct. The stack upgrade logic does not
interact with this since it's done via a custom command and will not trigger this hook.
:return:
"""
import params
if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.2') >= 0:
# try using the exact version first, falling back in just the stack if it's not defined
# which would only be during an intial cluster installation
version = params.current_version if params.current_version is not None else params.stack_version_unformatted
if not params.upgrade_suspended:
if params.host_sys_prepped:
Logger.warning("Skipping running stack-selector-tool for stack {0} as its a sys_prepped host. This may cause symlink pointers not to be created for HDP componets installed later on top of an already sys_prepped host.".format(version))
return
# On parallel command execution this should be executed by a single process at a time.
with FcntlBasedProcessLock(params.stack_select_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True):
stack_select.select_all(version)
def setup_hdp_symlinks():
"""
Invokes hdp-select set all against a calculated fully-qualified, "normalized" version based on a
stack version, such as "2.3". This should always be called after a component has been
installed to ensure that all HDP pointers are correct. The stack upgrade logic does not
interact with this since it's done via a custom command and will not trigger this hook.
:return:
"""
import params
if params.hdp_stack_version != "" and compare_versions(
params.hdp_stack_version, '2.2') >= 0:
# try using the exact version first, falling back in just the stack if it's not defined
# which would only be during an intial cluster installation
version = params.current_version if params.current_version is not None else params.stack_version_unformatted
if not params.upgrade_suspended:
# On parallel command execution this should be executed by a single process at a time.
with FcntlBasedProcessLock(
params.hdp_select_lock_file,
enabled=params.is_parallel_execution_enabled,
skip_fcntl_failures=True):
hdp_select.select_all(version)