def actionexecute(self, env): config = Script.get_config() version = default('/commandParams/version', None) stack_name = default('/hostLevelParams/stack_name', "") if not version: raise Fail("Value is required for '/commandParams/version'") # other os? if OSCheck.is_redhat_family(): cmd = ('/usr/bin/yum', 'clean', 'all') code, out = shell.call(cmd, sudo=True) real_ver = format_stack_version(version) if real_ver and check_stack_feature(StackFeature.ROLLING_UPGRADE, real_ver): stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME) cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all', version) code, out = shell.call(cmd, sudo=True) if code != 0: raise Exception("Command '{0}' exit code is nonzero".format(cmd)) if real_ver and check_stack_feature(StackFeature.CONFIG_VERSIONING, real_ver): # backup the old and symlink /etc/[component]/conf to <stack-root>/current/[component] for k, v in conf_select.get_package_dirs().iteritems(): for dir_def in v: link_config(dir_def['conf_dir'], dir_def['current_dir'])
def actionexecute(self, env): config = Script.get_config() version = default('/commandParams/version', None) stack_name = default('/hostLevelParams/stack_name', "") if not version: raise Fail("Value is required for '/commandParams/version'") # other os? if OSCheck.is_redhat_family(): cmd = ('/usr/bin/yum', 'clean', 'all') code, out = shell.call(cmd, sudo=True) min_ver = format_stack_version("2.2") real_ver = format_stack_version(version) if stack_name == "HDP": if compare_versions(real_ver, min_ver) >= 0: stack_selector_path = stack_tools.get_stack_tool_path( stack_tools.STACK_SELECTOR_NAME) cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all', version) code, out = shell.call(cmd, sudo=True) if compare_versions(real_ver, format_stack_version("2.3")) >= 0: # backup the old and symlink /etc/[component]/conf to <stack-root>/current/[component] for k, v in conf_select.get_package_dirs().iteritems(): for dir_def in v: link_config(dir_def['conf_dir'], dir_def['current_dir'])
def _create_config_links_if_necessary(self, stack_id, stack_version): """ Sets up the required structure for /etc/<component>/conf symlinks and <stack-root>/current configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3 stack_id: stack id, ie HDP-2.3 stack_version: version to set, ie 2.3.0.0-1234 """ if stack_id is None: Logger.info("Cannot create config links when stack_id is not defined") return args = stack_id.upper().split('-') if len(args) != 2: Logger.info("Unrecognized stack id {0}, cannot create config links".format(stack_id)) return target_stack_version = args[1] if not (target_stack_version and check_stack_feature(StackFeature.CONFIG_VERSIONING, target_stack_version)): Logger.info("Configuration symlinks are not needed for {0}".format(stack_version)) return # After upgrading hdf-select package from HDF-2.X to HDF-3.Y, we need to create this symlink if self.stack_name.upper() == "HDF" \ and not os.path.exists("/usr/bin/conf-select") and os.path.exists("/usr/bin/hdfconf-select"): Link("/usr/bin/conf-select", to = "/usr/bin/hdfconf-select") for package_name, directories in conf_select.get_package_dirs().iteritems(): conf_selector_name = stack_tools.get_stack_tool_name(stack_tools.CONF_SELECTOR_NAME) Logger.info("The current cluster stack of {0} does not require backing up configurations; " "only {1} versioned config directories will be created.".format(stack_version, conf_selector_name)) # only link configs for all known packages conf_select.select(self.stack_name, package_name, stack_version, ignore_errors = True)
def _create_config_links_if_necessary(self, stack_id, stack_version): """ Sets up the required structure for /etc/<component>/conf symlinks and <stack-root>/current configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3 stack_id: stack id, ie HDP-2.3 stack_version: version to set, ie 2.3.0.0-1234 """ if stack_id is None: Logger.info("Cannot create config links when stack_id is not defined") return args = stack_id.upper().split('-') if len(args) != 2: Logger.info("Unrecognized stack id {0}, cannot create config links".format(stack_id)) return target_stack_version = args[1] if not (target_stack_version and check_stack_feature(StackFeature.CONFIG_VERSIONING, target_stack_version)): Logger.info("Configuration symlinks are not needed for {0}".format(stack_version)) return for package_name, directories in conf_select.get_package_dirs().iteritems(): # if already on HDP 2.3, then we should skip making conf.backup folders if self.current_stack_version_formatted and check_stack_feature(StackFeature.CONFIG_VERSIONING, self.current_stack_version_formatted): conf_selector_name = stack_tools.get_stack_tool_name(stack_tools.CONF_SELECTOR_NAME) Logger.info("The current cluster stack of {0} does not require backing up configurations; " "only {1} versioned config directories will be created.".format(stack_version, conf_selector_name)) # only link configs for all known packages conf_select.select(self.stack_name, package_name, stack_version, ignore_errors = True) else: # link configs and create conf.backup folders for all known packages # this will also call conf-select select conf_select.convert_conf_directories_to_symlinks(package_name, stack_version, directories, skip_existing_links = False, link_to = "backup")
def actionexecute(self, env): version = default('/commandParams/version', None) if not version: raise Fail("Value is required for '/commandParams/version'") # other os? if OSCheck.is_redhat_family(): cmd = ('/usr/bin/yum', 'clean', 'all') code, out = shell.call(cmd, sudo=True) formatted_version = format_stack_version(version) if not formatted_version: raise Fail("Unable to determine a properly formatted stack version from {0}".format(version)) stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME) # this script runs on all hosts; if this host doesn't have stack components, # then don't invoke the stack tool # (no need to log that it's skipped - the function will do that) if is_host_skippable(stack_selector_path, formatted_version): return # invoke "set all" cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all', version) code, out = shell.call(cmd, sudo=True) if code != 0: raise Exception("Command '{0}' exit code is nonzero".format(cmd)) if check_stack_feature(StackFeature.CONFIG_VERSIONING, formatted_version): # backup the old and symlink /etc/[component]/conf to <stack-root>/current/[component] for k, v in conf_select.get_package_dirs().iteritems(): for dir_def in v: link_config(dir_def['conf_dir'], dir_def['current_dir'])
def create_config_version(self, env): import params for package_name, directories in conf_select.get_package_dirs( ).iteritems(): if package_name == 'registry': conf_select.convert_conf_directories_to_symlinks( package_name, params.current_version, directories)
def link_configs(struct_out_file): """ Use the conf_select module to link configuration directories correctly. """ import params json_version = load_version(struct_out_file) if not json_version: Logger.info( "Could not load 'version' from {0}".format(struct_out_file)) return if not params.sysprep_skip_conf_select or not os.path.exists( params.conf_select_marker_file): # On parallel command execution this should be executed by a single process at a time. with FcntlBasedProcessLock( params.link_configs_lock_file, enabled=params.is_parallel_execution_enabled, skip_fcntl_failures=True): for package_name, directories in conf_select.get_package_dirs( ).iteritems(): conf_select.convert_conf_directories_to_symlinks( package_name, json_version, directories) # create a file to mark that conf-selects were already done with open(params.conf_select_marker_file, "wb") as fp: pass else: Logger.info( format( "Skipping conf-select stage, since cluster-env/sysprep_skip_conf_select is set and mark file {conf_select_marker_file} exists" ))
def unlink_all_configs(self, env): """ Reverses the work performed in link_config. This should only be used when downgrading from HDP 2.3 to 2.2 in order to under the symlink work required for 2.3. """ stack_name = default('/hostLevelParams/stack_name', "").upper() downgrade_to_version = default('/commandParams/version', None) downgrade_from_version = default( '/commandParams/downgrade_from_version', None) upgrade_direction = default("/commandParams/upgrade_direction", Direction.UPGRADE) # downgrade only if upgrade_direction != Direction.DOWNGRADE: Logger.warning( "Unlinking configurations should only be performed on a downgrade." ) return # HDP only if stack_name != "HDP": Logger.warning( "Unlinking configurations should only be performed on the HDP stack." ) return if downgrade_to_version is None or downgrade_from_version is None: Logger.warning( "Both 'commandParams/version' and 'commandParams/downgrade_from_version' must be specified to unlink configs on downgrade." ) return Logger.info( "Unlinking all configs when downgrading from HDP 2.3 to 2.2") # normalize the versions stack_23 = format_stack_version("2.3") downgrade_to_version = format_stack_version(downgrade_to_version) downgrade_from_version = format_stack_version(downgrade_from_version) # downgrade-to-version must be 2.2 (less than 2.3) if compare_versions(downgrade_to_version, stack_23) >= 0: Logger.warning( "Unlinking configurations should only be performed when downgrading to HDP 2.2" ) return # downgrade-from-version must be 2.3+ if compare_versions(downgrade_from_version, stack_23) < 0: Logger.warning( "Unlinking configurations should only be performed when downgrading from HDP 2.3 or later" ) return # iterate through all directory conf mappings and undo the symlinks for key, value in conf_select.get_package_dirs().iteritems(): for directory_mapping in value: original_config_directory = directory_mapping['conf_dir'] self._unlink_config(original_config_directory)
def _fix_default_links_for_current(self): """ If a prior version of Ambari did not correctly reverse the conf symlinks, then they would be put into a bad state when distributing a new stack. For example: /etc/component/conf (directory) <stack-root>/v1/component/conf -> /etc/component/conf When distributing v2, we'd detect the /etc/component/conf problems and would try to adjust it: /etc/component/conf -> <stack-root>/current/component/conf <stack-root>/v2/component/conf -> /etc/component/v2/0 The problem is that v1 never gets changed (since the stack being distributed is v2), and we end up with a circular link: /etc/component/conf -> <stack-root>/current/component/conf <stack-root>/v1/component/conf -> /etc/component/conf :return: None """ Logger.info( "Attempting to fix any configuration symlinks which are not in the correct state" ) from resource_management.libraries.functions import stack_select restricted_packages = conf_select.get_restricted_packages() if 0 == len(restricted_packages): Logger.info( "There are no restricted conf-select packages for this installation" ) else: Logger.info("Restricting conf-select packages to {0}".format( restricted_packages)) for package_name, directories in conf_select.get_package_dirs( ).iteritems(): Logger.info( "Attempting to fix the default conf links for {0}".format( package_name)) Logger.info( "The following directories will be fixed for {0}: {1}".format( package_name, str(directories))) component_name = None for directory_struct in directories: if "component" in directory_struct: component_name = directory_struct["component"] if component_name: stack_version = stack_select.get_stack_version_before_install( component_name) if 0 == len(restricted_packages ) or package_name in restricted_packages: if stack_version: conf_select.convert_conf_directories_to_symlinks( package_name, stack_version, directories) else: Logger.warning( "Unable to fix {0} since there is no known installed version for this component" .format(package_name))
def unlink_all_configs(self, env): """ Reverses the work performed in link_config. This should only be used when downgrading from HDP 2.3 to 2.2 in order to under the symlink work required for 2.3. """ stack_name = default('/hostLevelParams/stack_name', "").upper() downgrade_to_version = default('/commandParams/version', None) downgrade_from_version = default( '/commandParams/downgrade_from_version', None) upgrade_direction = default("/commandParams/upgrade_direction", Direction.UPGRADE) # downgrade only if upgrade_direction != Direction.DOWNGRADE: Logger.warning( "Unlinking configurations should only be performed on a downgrade." ) return if downgrade_to_version is None or downgrade_from_version is None: Logger.warning( "Both 'commandParams/version' and 'commandParams/downgrade_from_version' must be specified to unlink configs on downgrade." ) return Logger.info( "Unlinking all configs when downgrading from {0} {1} to {2}". format(stack_name, downgrade_from_version, downgrade_to_version)) # normalize the versions downgrade_to_version = format_stack_version(downgrade_to_version) downgrade_from_version = format_stack_version(downgrade_from_version) # downgrade-to-version must be 2.2 (less than 2.3) if downgrade_to_version and check_stack_feature( StackFeature.CONFIG_VERSIONING, downgrade_to_version): Logger.warning( "Unlinking configurations should not be performed when downgrading {0} {1} to {2}" .format(stack_name, downgrade_from_version, downgrade_to_version)) return # downgrade-from-version must be 2.3+ if not (downgrade_from_version and check_stack_feature( StackFeature.CONFIG_VERSIONING, downgrade_from_version)): Logger.warning( "Unlinking configurations should not be performed when downgrading {0} {1} to {2}" .format(stack_name, downgrade_from_version, downgrade_to_version)) return # iterate through all directory conf mappings and undo the symlinks for key, value in conf_select.get_package_dirs().iteritems(): for directory_mapping in value: original_config_directory = directory_mapping['conf_dir'] self._unlink_config(original_config_directory)
def test_hook_default_conf_select(self, rmtree_mock, symlink_mock, conf_select_select_mock, conf_select_create_mock): def mocked_conf_select(arg1, arg2, arg3, dry_run = False): return "/etc/{0}/{1}/0".format(arg2, arg3) conf_select_create_mock.side_effect = mocked_conf_select config_file = self.get_src_folder() + "/test/python/stacks/configs/default.json" with open(config_file, "r") as f: json_content = json.load(f) version = '2.3.0.0-1234' json_content['commandParams']['version'] = version json_content['clusterLevelParams']['stack_version'] = "2.3" self.executeScript("after-INSTALL/scripts/hook.py", classname="AfterInstallHook", command="hook", target=RMFTestCase.TARGET_STACK_HOOKS, config_dict = json_content, config_overrides = self.CONFIG_OVERRIDES) self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hive-server2', '2.3.0.0-1234'), sudo = True) self.assertResourceCalled('XmlConfig', 'core-site.xml', owner = 'hdfs', group = 'hadoop', conf_dir = "/usr/hdp/2.3.0.0-1234/hadoop/conf", configurations = self.getConfig()['configurations']['core-site'], configuration_attributes = self.getConfig()['configurationAttributes']['core-site'], only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf", xml_include_file=None) self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder/conf', mode = 0755, cd_access = 'a', create_parents = True) package_dirs = conf_select.get_package_dirs(); for package, dir_defs in package_dirs.iteritems(): for dir_def in dir_defs: conf_dir = dir_def['conf_dir'] conf_backup_dir = conf_dir + ".backup" current_dir = dir_def['current_dir'] self.assertResourceCalled('Execute', ('cp', '-R', '-p', conf_dir, conf_backup_dir), not_if = 'test -e ' + conf_backup_dir, sudo = True,) self.assertResourceCalled('Directory', conf_dir, action = ['delete'],) self.assertResourceCalled('Link', conf_dir, to = current_dir,) self.assertNoMoreResources()
def test_symlink_noop(self): """ Tests that conf-select symlinking does nothing if the directory doesn't exist :return: """ packages = conf_select.get_package_dirs() conf_select.convert_conf_directories_to_symlinks( "hadoop", "2.3.0.0-1234", packages["hadoop"]) self.assertEqual(pprint.pformat(self.env.resource_list), "[]")
def test_symlink_conversion_relinks_wrong_link(self): """ Tests that conf-select symlinking can detect a wrong directory :return: """ packages = conf_select.get_package_dirs() conf_select.convert_conf_directories_to_symlinks( "hadoop", "2.3.0.0-1234", packages["hadoop"]) self.assertEqual( pprint.pformat(self.env.resource_list), "[Link['/etc/hadoop/conf'], Link['/etc/hadoop/conf']]")
def test_symlink_conversion_to_current(self, islink_mock, path_mock, isdir_mock, shell_call_mock): """ Tests that conf-select creates the correct symlink directories. :return: """ def mock_call(command, **kwargs): """ Instead of shell.call, call a command whose output equals the command. :param command: Command that will be echoed. :return: Returns a tuple of (process output code, stdout, stderr) """ return (0, "/etc/hadoop/conf", None) def path_mock_call(path): if path == "/etc/hadoop/conf": return True if path == "/etc/hadoop/2.3.0.0-1234/0": return True return False def islink_mock_call(path): if path == "/etc/hadoop/conf": return False return False def isdir_mock_call(path): if path == "/etc/hadoop/conf": return True return False packages = conf_select.get_package_dirs() path_mock.side_effect = path_mock_call islink_mock.side_effect = islink_mock_call shell_call_mock.side_effect = mock_call conf_select.convert_conf_directories_to_symlinks( "hadoop", "2.3.0.0-1234", packages["hadoop"]) self.assertEqual( pprint.pformat(self.env.resource_list[0]), "Execute[('cp', '-R', '-p', u'/etc/hadoop/conf', u'/etc/hadoop/conf.backup')]" ) self.assertEqual(pprint.pformat(self.env.resource_list[1]), "Directory['/etc/hadoop/conf']") self.assertEqual(pprint.pformat(self.env.resource_list[2]), "Link['/etc/hadoop/conf']")
def _relink_configurations_with_conf_select(self, stack_id, stack_version): """ Sets up the required structure for /etc/<component>/conf symlinks and <stack-root>/current configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3 stack_id: stack id, ie HDP-2.3 stack_version: version to set, ie 2.3.0.0-1234 """ if stack_id is None: Logger.info( "Cannot create config links when stack_id is not defined") return args = stack_id.upper().split('-') if len(args) != 2: Logger.info( "Unrecognized stack id {0}, cannot create config links".format( stack_id)) return target_stack_version = args[1] if not (target_stack_version and check_stack_feature( StackFeature.CONFIG_VERSIONING, target_stack_version)): Logger.info("Configuration symlinks are not needed for {0}".format( stack_version)) return # After upgrading hdf-select package from HDF-2.X to HDF-3.Y, we need to create this symlink if self.stack_name.upper() == "HDF" \ and not sudo.path_exists("/usr/bin/conf-select") and sudo.path_exists("/usr/bin/hdfconf-select"): Link("/usr/bin/conf-select", to="/usr/bin/hdfconf-select") restricted_packages = conf_select.get_restricted_packages() if 0 == len(restricted_packages): Logger.info( "There are no restricted conf-select packages for this installation" ) else: Logger.info("Restricting conf-select packages to {0}".format( restricted_packages)) for package_name, directories in conf_select.get_package_dirs( ).iteritems(): if 0 == len(restricted_packages ) or package_name in restricted_packages: conf_select.convert_conf_directories_to_symlinks( package_name, stack_version, directories)
def link_configs(struct_out_file): """ Links configs, only on a fresh install of HDP-2.3 and higher """ import params json_version = load_version(struct_out_file) if not json_version: Logger.info("Could not load 'version' from {0}".format(struct_out_file)) return # On parallel command execution this should be executed by a single process at a time. with FcntlBasedProcessLock(params.link_configs_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True): for k, v in conf_select.get_package_dirs().iteritems(): conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def link_configs(struct_out_file): """ Use the conf_select module to link configuration directories correctly. """ import params json_version = load_version(struct_out_file) if not json_version: Logger.info("Could not load 'version' from {0}".format(struct_out_file)) return # On parallel command execution this should be executed by a single process at a time. with FcntlBasedProcessLock(params.link_configs_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True): for package_name, directories in conf_select.get_package_dirs().iteritems(): conf_select.convert_conf_directories_to_symlinks(package_name, json_version, directories)
def test_symlink_conversion_bad_linkto(self): """ Tests that a bad enum throws an exception. :return: """ packages = conf_select.get_package_dirs() try: conf_select.convert_conf_directories_to_symlinks( "hadoop", "2.3.0.0-1234", packages["hadoop"], link_to="INVALID") raise Exception( "Expected failure when supplying a bad enum for link_to") except: pass
def link_configs(struct_out_file): """ Links configs, only on a fresh install of BigInsights-4.1 and higher """ if not Script.is_stack_greater_or_equal("4.1"): Logger.info("Can only link configs for BigInsights-4.1 and higher.") return json_version = load_version(struct_out_file) if not json_version: Logger.info( "Could not load 'version' from {0}".format(struct_out_file)) return for k, v in conf_select.get_package_dirs().iteritems(): conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def _fix_default_links(self, package_name, component_name): """ If a prior version of Ambari did not correctly reverse the conf symlinks, then they would be put into a bad state when distributing a new stack. For example: /etc/component/conf (directory) <stack-root>/v1/component/conf -> /etc/component/conf When distributing v2, we'd detect the /etc/component/conf problems and would try to adjust it: /etc/component/conf -> <stack-root>/current/component/conf <stack-root>/v2/component/conf -> /etc/component/v2/0 The problem is that v1 never gets changed (since the stack being distributed is v2), and we end up with a circular link: /etc/component/conf -> <stack-root>/current/component/conf <stack-root>/v1/component/conf -> /etc/component/conf :return: None """ from resource_management.libraries.functions import stack_select package_dirs = conf_select.get_package_dirs() if package_name in package_dirs: Logger.info( "Determining if the default conf links for {0} need to be fixed" .format(package_name)) directories = package_dirs[package_name] Logger.info( "The following directories will be checked for {0}: {1}". format(package_name, str(directories))) stack_version = stack_select.get_stack_version_before_install( component_name) if stack_version: conf_select.convert_conf_directories_to_symlinks( package_name, stack_version, directories)
def test_downgrade_unlink_configs(self, family_mock, get_config_mock, call_mock, isdir_mock, islink_mock): """ Tests downgrading from 2.3 to 2.2 to ensure that conf symlinks are removed and the backup directories restored. """ isdir_mock.return_value = True # required for the test to run since the Execute calls need this from resource_management.core.environment import Environment env = Environment(test_mode=True) with env: # Mock the config objects json_file_path = os.path.join(self.get_custom_actions_dir(), "ru_execute_tasks_namenode_prepare.json") self.assertTrue(os.path.isfile(json_file_path)) with open(json_file_path, "r") as json_file: json_payload = json.load(json_file) # alter JSON for a downgrade from 2.3 to 2.2 json_payload['commandParams']['version'] = "2.2.0.0-1234" json_payload['commandParams']['downgrade_from_version'] = "2.3.0.0-1234" json_payload['commandParams']['original_stack'] = "HDP-2.2" json_payload['commandParams']['target_stack'] = "HDP-2.3" json_payload['commandParams']['upgrade_direction'] = "downgrade" json_payload['hostLevelParams']['stack_version'] = "2.2" config_dict = ConfigDictionary(json_payload) family_mock.return_value = True get_config_mock.return_value = config_dict call_mock.side_effect = fake_call # echo the command # test the function ru_execute = UpgradeSetAll() ru_execute.unlink_all_configs(None) # verify that os.path.islink was called for each conf self.assertTrue(islink_mock.called) for key, value in conf_select.get_package_dirs().iteritems(): for directory_mapping in value: original_config_directory = directory_mapping['conf_dir'] is_link_called = False for call in islink_mock.call_args_list: call_tuple = call[0] if original_config_directory in call_tuple: is_link_called = True if not is_link_called: self.fail("os.path.islink({0}) was never called".format(original_config_directory)) # alter JSON for a downgrade from 2.3 to 2.3 with open(json_file_path, "r") as json_file: json_payload = json.load(json_file) json_payload['commandParams']['version'] = "2.3.0.0-1234" json_payload['commandParams']['downgrade_from_version'] = "2.3.0.0-5678" json_payload['commandParams']['original_stack'] = "HDP-2.3" json_payload['commandParams']['target_stack'] = "HDP-2.3" json_payload['commandParams']['upgrade_direction'] = "downgrade" json_payload['hostLevelParams']['stack_version'] = "2.3" # reset config config_dict = ConfigDictionary(json_payload) family_mock.return_value = True get_config_mock.return_value = config_dict # reset mock islink_mock.reset_mock() # test the function ru_execute = UpgradeSetAll() ru_execute.unlink_all_configs(None) # ensure it wasn't called this time self.assertFalse(islink_mock.called) with open(json_file_path, "r") as json_file: json_payload = json.load(json_file) # alter JSON for a downgrade from 2.2 to 2.2 json_payload['commandParams']['version'] = "2.2.0.0-1234" json_payload['commandParams']['downgrade_from_version'] = "2.2.0.0-5678" json_payload['commandParams']['original_stack'] = "HDP-2.2" json_payload['commandParams']['target_stack'] = "HDP-2.2" json_payload['commandParams']['upgrade_direction'] = "downgrade" json_payload['hostLevelParams']['stack_version'] = "2.2" # reset config config_dict = ConfigDictionary(json_payload) family_mock.return_value = True get_config_mock.return_value = config_dict # reset mock islink_mock.reset_mock() # test the function ru_execute = UpgradeSetAll() ru_execute.unlink_all_configs(None) # ensure it wasn't called this time self.assertFalse(islink_mock.called)
def test_hook_default_conf_select_with_error(self, rmtree_mock, symlink_mock, conf_select_select_mock, conf_select_create_mock): def mocked_conf_select(arg1, arg2, arg3, dry_run = False, ignore_errors = False): if arg2 == "pig" and not dry_run: if not ignore_errors: raise Exception("whoops") else: return None return "/etc/{0}/{1}/0".format(arg2, arg3) conf_select_create_mock.side_effect = mocked_conf_select conf_select_select_mock.side_effect = mocked_conf_select config_file = self.get_src_folder() + "/test/python/stacks/2.0.6/configs/default.json" with open(config_file, "r") as f: json_content = json.load(f) version = '2.3.0.0-1234' json_content['commandParams']['version'] = version json_content['hostLevelParams']['stack_version'] = "2.3" self.executeScript("2.0.6/hooks/after-INSTALL/scripts/hook.py", classname="AfterInstallHook", command="hook", config_dict = json_content, config_overrides = self.CONFIG_OVERRIDES) self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hive-server2', '2.3.0.0-1234'), sudo = True) self.assertResourceCalled('XmlConfig', 'core-site.xml', owner = 'hdfs', group = 'hadoop', conf_dir = "/usr/hdp/current/hadoop-client/conf", configurations = self.getConfig()['configurations']['core-site'], configuration_attributes = self.getConfig()['configuration_attributes']['core-site'], only_if="ls /usr/hdp/current/hadoop-client/conf") self.assertResourceCalled('Directory', '/etc/ambari-logsearch-logfeeder/conf', mode = 0755, cd_access = 'a', create_parents = True) package_dirs = conf_select.get_package_dirs(); for package, dir_defs in package_dirs.iteritems(): for dir_def in dir_defs: conf_dir = dir_def['conf_dir'] conf_backup_dir = conf_dir + ".backup" self.assertResourceCalled('Execute', ('cp', '-R', '-p', conf_dir, conf_backup_dir), not_if = 'test -e ' + conf_backup_dir, sudo = True,) for dir_def in dir_defs: conf_dir = dir_def['conf_dir'] current_dir = dir_def['current_dir'] self.assertResourceCalled('Directory', conf_dir, action = ['delete'],) self.assertResourceCalled('Link', conf_dir, to = current_dir,) #HACK for Atlas if package in ["atlas", ]: self.assertResourceCalled('Execute', 'ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E cp -R --no-clobber /etc/atlas/conf.backup/* /etc/atlas/conf', only_if = 'test -e ' + "/etc/atlas/conf") self.assertNoMoreResources()
def test_hook_default_conf_select(self, rmtree_mock, symlink_mock, conf_select_select_mock, conf_select_create_mock): def mocked_conf_select(arg1, arg2, arg3, dry_run=False): return "/etc/{0}/{1}/0".format(arg2, arg3) conf_select_create_mock.side_effect = mocked_conf_select config_file = self.get_src_folder( ) + "/test/python/stacks/2.0.6/configs/default.json" with open(config_file, "r") as f: json_content = json.load(f) version = '2.3.0.0-1234' json_content['commandParams']['version'] = version json_content['hostLevelParams']['stack_version'] = "2.3" self.executeScript("2.0.6/hooks/after-INSTALL/scripts/hook.py", classname="AfterInstallHook", command="hook", config_dict=json_content) self.assertResourceCalled( 'Execute', 'ambari-sudo.sh /usr/bin/hdp-select set all `ambari-python-wrap /usr/bin/hdp-select versions | grep ^2.3 | tail -1`', only_if='ls -d /usr/hdp/2.3*') self.assertResourceCalled( 'XmlConfig', 'core-site.xml', owner='hdfs', group='hadoop', conf_dir="/usr/hdp/current/hadoop-client/conf", configurations=self.getConfig()['configurations']['core-site'], configuration_attributes=self.getConfig() ['configuration_attributes']['core-site'], only_if="ls /usr/hdp/current/hadoop-client/conf") package_dirs = conf_select.get_package_dirs() for package, dir_defs in package_dirs.iteritems(): for dir_def in dir_defs: conf_dir = dir_def['conf_dir'] conf_backup_dir = conf_dir + ".backup" self.assertResourceCalled( 'Execute', ('cp', '-R', '-p', conf_dir, conf_backup_dir), not_if='test -e ' + conf_backup_dir, sudo=True, ) for dir_def in dir_defs: conf_dir = dir_def['conf_dir'] current_dir = dir_def['current_dir'] self.assertResourceCalled( 'Directory', conf_dir, action=['delete'], ) self.assertResourceCalled( 'Link', conf_dir, to=current_dir, ) #HACK for Atlas if package in [ "atlas", ]: self.assertResourceCalled( 'Execute', 'ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E cp -R --no-clobber /etc/atlas/conf.backup/* /etc/atlas/conf', only_if='test -e ' + "/etc/atlas/conf") self.assertNoMoreResources()
def test_hook_default_conf_select_suspended(self, rmtree_mock, symlink_mock, conf_select_select_mock, conf_select_create_mock): def mocked_conf_select(arg1, arg2, arg3, dry_run=False): return "/etc/{0}/{1}/0".format(arg2, arg3) conf_select_create_mock.side_effect = mocked_conf_select config_file = self.get_src_folder( ) + "/test/python/stacks/2.0.6/configs/default.json" with open(config_file, "r") as f: json_content = json.load(f) version = '2.3.0.0-1234' json_content['commandParams']['version'] = version json_content['hostLevelParams']['stack_version'] = "2.3" json_content['roleParams']['upgrade_suspended'] = "true" self.executeScript("2.0.6/hooks/after-INSTALL/scripts/hook.py", classname="AfterInstallHook", command="hook", config_dict=json_content, config_overrides=self.CONFIG_OVERRIDES) # same assertions as test_hook_default_conf_select, but skip hdp-select set all self.assertResourceCalled( 'XmlConfig', 'core-site.xml', owner='hdfs', group='hadoop', conf_dir="/usr/hdp/2.3.0.0-1234/hadoop/conf", configurations=self.getConfig()['configurations']['core-site'], configuration_attributes=self.getConfig() ['configuration_attributes']['core-site'], only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf") package_dirs = conf_select.get_package_dirs() for package, dir_defs in package_dirs.iteritems(): for dir_def in dir_defs: conf_dir = dir_def['conf_dir'] conf_backup_dir = conf_dir + ".backup" current_dir = dir_def['current_dir'] self.assertResourceCalled( 'Execute', ('cp', '-R', '-p', conf_dir, conf_backup_dir), not_if='test -e ' + conf_backup_dir, sudo=True, ) self.assertResourceCalled( 'Directory', conf_dir, action=['delete'], ) self.assertResourceCalled( 'Link', conf_dir, to=current_dir, ) self.assertNoMoreResources()