def setup_stack_symlinks(struct_out_file): """ Invokes <stack-selector-tool> set all against a calculated fully-qualified, "normalized" version based on a stack version, such as "2.3". This should always be called after a component has been installed to ensure that all HDP pointers are correct. The stack upgrade logic does not interact with this since it's done via a custom command and will not trigger this hook. :return: """ import params if params.upgrade_suspended: Logger.warning("Skipping running stack-selector-tool because there is a suspended upgrade") return if params.host_sys_prepped: Logger.warning("Skipping running stack-selector-tool becase this is a sys_prepped host. This may cause symlink pointers not to be created for HDP componets installed later on top of an already sys_prepped host.") return # get the packages which the stack-select tool should be used on stack_select_packages = stack_select.get_packages(stack_select.PACKAGE_SCOPE_INSTALL) if stack_select_packages is None: return json_version = load_version(struct_out_file) if not json_version: Logger.info("There is no advertised version for this component stored in {0}".format(struct_out_file)) return # On parallel command execution this should be executed by a single process at a time. with FcntlBasedProcessLock(params.stack_select_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True): for package in stack_select_packages: stack_select.select(package, json_version)
def locking_configure(obj, *args, **kw): # local import to avoid circular dependency (default imports Script) from resource_management.libraries.functions.default import default parallel_execution_enabled = int(default("/agentConfigParams/agent/parallel_execution", 0)) == 1 lock = FcntlBasedProcessLock(get_config_lock_file(), skip_fcntl_failures = True, enabled = parallel_execution_enabled) with lock: original_configure(obj, *args, **kw)
def link_configs(struct_out_file): """ Use the conf_select module to link configuration directories correctly. """ import params json_version = load_version(struct_out_file) if not json_version: Logger.info( "Could not load 'version' from {0}".format(struct_out_file)) return if not params.sysprep_skip_conf_select or not os.path.exists( params.conf_select_marker_file): # On parallel command execution this should be executed by a single process at a time. with FcntlBasedProcessLock( params.link_configs_lock_file, enabled=params.is_parallel_execution_enabled, skip_fcntl_failures=True): for package_name, directories in conf_select.get_package_dirs( ).iteritems(): conf_select.convert_conf_directories_to_symlinks( package_name, json_version, directories) # create a file to mark that conf-selects were already done with open(params.conf_select_marker_file, "wb") as fp: pass else: Logger.info( format( "Skipping conf-select stage, since cluster-env/sysprep_skip_conf_select is set and mark file {conf_select_marker_file} exists" ))
def dummy_task(index, mutex): with FcntlBasedProcessLock(lock_file, skip_fcntl_failures=False): if (not mutex.acquire(block=False)): raise Exception( "ERROR: FcntlBasedProcessLock was acquired by several processes" ) time.sleep(0.1) mutex.release()
def link_configs(struct_out_file): """ Use the conf_select module to link configuration directories correctly. """ import params json_version = load_version(struct_out_file) if not json_version: Logger.info("Could not load 'version' from {0}".format(struct_out_file)) return # On parallel command execution this should be executed by a single process at a time. with FcntlBasedProcessLock(params.link_configs_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True): for package_name, directories in conf_select.get_package_dirs().iteritems(): conf_select.convert_conf_directories_to_symlinks(package_name, json_version, directories)
def link_configs(struct_out_file): """ Links configs, only on a fresh install of HDP-2.3 and higher """ import params json_version = load_version(struct_out_file) if not json_version: Logger.info("Could not load 'version' from {0}".format(struct_out_file)) return # On parallel command execution this should be executed by a single process at a time. with FcntlBasedProcessLock(params.link_configs_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True): for k, v in conf_select.get_package_dirs().iteritems(): conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def setup_stack_symlinks(): """ Invokes <stack-selector-tool> set all against a calculated fully-qualified, "normalized" version based on a stack version, such as "2.3". This should always be called after a component has been installed to ensure that all HDP pointers are correct. The stack upgrade logic does not interact with this since it's done via a custom command and will not trigger this hook. :return: """ import params if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.2') >= 0: # try using the exact version first, falling back in just the stack if it's not defined # which would only be during an intial cluster installation version = params.current_version if params.current_version is not None else params.stack_version_unformatted if not params.upgrade_suspended: if params.host_sys_prepped: Logger.warning("Skipping running stack-selector-tool for stack {0} as its a sys_prepped host. This may cause symlink pointers not to be created for HDP componets installed later on top of an already sys_prepped host.".format(version)) return # On parallel command execution this should be executed by a single process at a time. with FcntlBasedProcessLock(params.stack_select_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True): stack_select.select_all(version)
def setup_hdp_symlinks(): """ Invokes hdp-select set all against a calculated fully-qualified, "normalized" version based on a stack version, such as "2.3". This should always be called after a component has been installed to ensure that all HDP pointers are correct. The stack upgrade logic does not interact with this since it's done via a custom command and will not trigger this hook. :return: """ import params if params.hdp_stack_version != "" and compare_versions( params.hdp_stack_version, '2.2') >= 0: # try using the exact version first, falling back in just the stack if it's not defined # which would only be during an intial cluster installation version = params.current_version if params.current_version is not None else params.stack_version_unformatted if not params.upgrade_suspended: # On parallel command execution this should be executed by a single process at a time. with FcntlBasedProcessLock( params.hdp_select_lock_file, enabled=params.is_parallel_execution_enabled, skip_fcntl_failures=True): hdp_select.select_all(version)