def actionexecute(self, env): resolve_ambari_config() # Parse parameters from command json file. config = Script.get_config() host_name = socket.gethostname() version = default('/roleParams/version', None) # These 2 variables are optional service_package_folder = default('/roleParams/service_package_folder', None) hooks_folder = default('/roleParams/hooks_folder', None) tasks = json.loads(config['roleParams']['tasks']) if tasks: for t in tasks: task = ExecuteTask(t) Logger.info(str(task)) # If a (script, function) exists, it overwrites the command. if task.script and task.function and service_package_folder and hooks_folder: file_cache = FileCache(agent_config) command_paths = {"commandParams": {"service_package_folder": service_package_folder, "hooks_folder": hooks_folder } } server_url_prefix = default('/hostLevelParams/jdk_location', "") base_dir = file_cache.get_service_base_dir(command_paths, server_url_prefix) script_path = os.path.join(base_dir, task.script) if not os.path.exists(script_path): message = "Script %s does not exist" % str(script_path) raise Fail(message) # Notice that the script_path is now the fully qualified path, and the # same command-#.json file is used. # Also, the python wrapper is used, since it sets up the correct environment variables command_params = ["/usr/bin/ambari-python-wrap", script_path, task.function, self.command_data_file, self.basedir, self.stroutfile, self.logging_level, Script.get_tmp_dir()] task.command = " ".join(command_params) # Replace redundant whitespace to make the unit tests easier to validate task.command = re.sub("\s+", " ", task.command).strip() if task.command: task.command = replace_variables(task.command, host_name, version) code, out = shell.call(task.command) Logger.info("Command: %s\nCode: %s, Out: %s" % (task.command, str(code), str(out))) if code != 0: raise Fail(out)
def test_struct_out(self): from resource_management.libraries.script import Script configs_path = os.path.join(RMFTestCase._getSrcFolder(), "test/python/stacks", self.STACK_VERSION, "configs") script = Script() script.stroutfile = os.path.join(configs_path, "structured-out-status.json") script.load_structured_out() self.assertFalse("version" in script.structuredOut)
def falcon(type, action = None): import params if action == 'config': env = Environment.get_instance() # These 2 parameters are used in ../templates/client.properties.j2 env.config.params["falcon_host"] = params.falcon_host env.config.params["falcon_port"] = params.falcon_port File(os.path.join(params.falcon_conf_dir, 'falcon-env.sh'), content = InlineTemplate(params.falcon_env_sh_template)) File(os.path.join(params.falcon_conf_dir, 'client.properties'), content = Template('client.properties.j2')) PropertiesFile(os.path.join(params.falcon_conf_dir, 'runtime.properties'), properties = params.falcon_runtime_properties) PropertiesFile(os.path.join(params.falcon_conf_dir, 'startup.properties'), properties = params.falcon_startup_properties) if type == 'server': ServiceConfig(params.falcon_win_service_name, action = "change_user", username = params.falcon_user, password = Script.get_password(params.falcon_user)) if action == 'start': Service(params.falcon_win_service_name, action = "start") if action == 'stop': Service(params.falcon_win_service_name, action = "stop")
def actionexecute(self, env): config = Script.get_config() version = default('/commandParams/version', None) stack_name = default('/hostLevelParams/stack_name', "") if not version: raise Fail("Value is required for '/commandParams/version'") # other os? if OSCheck.is_redhat_family(): cmd = ('/usr/bin/yum', 'clean', 'all') code, out = shell.call(cmd, sudo=True) min_ver = format_hdp_stack_version("2.2") real_ver = format_hdp_stack_version(version) if stack_name == "HDP": if compare_versions(real_ver, min_ver) >= 0: cmd = ('hdp-select', 'set', 'all', version) code, out = shell.call(cmd, sudo=True) if compare_versions(real_ver, format_hdp_stack_version("2.3")) >= 0: # backup the old and symlink /etc/[component]/conf to /usr/hdp/current/[component] for k, v in conf_select.PACKAGE_DIRS.iteritems(): for dir_def in v: link_config(dir_def['conf_dir'], dir_def['current_dir'])
def check_indexer_parameters(): """ Ensure that all required parameters have been defined for the chosen Indexer; either Solr or Elasticsearch. """ missing = [] config = Script.get_config() indexer = config['configurations']['metron-indexing-env']['ra_indexing_writer'] Logger.info('Checking parameters for indexer = ' + indexer) if indexer == 'Solr': # check for all required solr parameters if not config['configurations']['metron-env']['solr_zookeeper_url']: missing.append("metron-env/solr_zookeeper_url") else: # check for all required elasticsearch parameters if not config['configurations']['metron-env']['es_cluster_name']: missing.append("metron-env/es_cluster_name") if not config['configurations']['metron-env']['es_hosts']: missing.append("metron-env/es_hosts") if not config['configurations']['metron-env']['es_date_format']: missing.append("metron-env/es_date_format") if len(missing) > 0: raise Fail("Missing required indexing parameters(s): indexer={0}, missing={1}".format(indexer, missing))
def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) if Script.is_hdp_stack_greater_or_equal('2.3.0.0'): conf_select.select(params.stack_name, "hadoop", params.version) hdp_select.select("hadoop-hdfs-nfs3", params.version)
def should_install_mysql(): config = Script.get_config() hive_database = config['configurations']['hive-env']['hive_database'] hive_use_existing_db = hive_database.startswith('Existing') if hive_use_existing_db: return False return _has_applicable_local_component(config, "MYSQL_SERVER")
def should_install_falcon_atlas_hook(): config = Script.get_config() stack_version_unformatted = config['hostLevelParams']['stack_version'] stack_version_formatted = format_stack_version(stack_version_unformatted) if check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT_2_3, stack_version_formatted) \ or check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT, stack_version_formatted): return _has_applicable_local_component(config, ['FALCON_SERVER']) return False
def pre_rolling_restart(self, env): import params env.set_params(params) if Script.is_hdp_stack_greater_or_equal("2.3"): # phoenix uses hbase configs conf_select.select(params.stack_name, "hbase", params.version) hdp_select.select("phoenix-server", params.version)
def test_install_packages(self, package_provider_mock): no_packages_config = { 'hostLevelParams' : { 'repo_info' : "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]", 'agent_stack_retry_count': '5', 'agent_stack_retry_on_unavailability': 'false' } } empty_config = { 'hostLevelParams' : { 'package_list' : '', 'repo_info' : "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]", 'agent_stack_retry_count': '5', 'agent_stack_retry_on_unavailability': 'false' } } dummy_config = { 'hostLevelParams' : { 'package_list' : "[{\"type\":\"rpm\",\"name\":\"hbase\", \"condition\": \"\"}," "{\"type\":\"rpm\",\"name\":\"yet-another-package\", \"condition\": \"\"}]", 'repo_info' : "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]", 'service_repo_info' : "[{\"mirrorsList\":\"abc\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]", 'agent_stack_retry_count': '5', 'agent_stack_retry_on_unavailability': 'false' } } # Testing config without any keys with Environment(".", test_mode=True) as env: script = Script() Script.config = no_packages_config script.install_packages(env) resource_dump = pprint.pformat(env.resource_list) self.assertEquals(resource_dump, "[]") # Testing empty package list with Environment(".", test_mode=True) as env: script = Script() Script.config = empty_config script.install_packages(env) resource_dump = pprint.pformat(env.resource_list) self.assertEquals(resource_dump, "[]") # Testing installing of a list of packages with Environment(".", test_mode=True) as env: script = Script() Script.config = dummy_config script.install_packages("env") resource_dump = pprint.pformat(env.resource_list) self.assertEqual(resource_dump, '[Package[\'hbase\'], Package[\'yet-another-package\']]')
def pre_rolling_restart(self, env): Logger.info("Executing Metastore Rolling Upgrade pre-restart") import params env.set_params(params) if Script.is_hdp_stack_greater_or_equal("2.3"): self.upgrade_schema(env) if params.version and compare_versions(format_hdp_stack_version(params.version), "2.2.0.0") >= 0: conf_select.select(params.stack_name, "hive", params.version) hdp_select.select("hive-metastore", params.version)
def default(name, default_value): subdicts = filter(None, name.split('/')) curr_dict = Script.get_config() if not curr_dict: return default_value for x in subdicts: if x in curr_dict: curr_dict = curr_dict[x] else: return default_value return curr_dict
def pre_rolling_restart(self, env): import params env.set_params(params) # this function should not execute if the version can't be determined or # is not at least HDP 2.2.0.0 if Script.is_hdp_stack_less_than("2.2"): return Logger.info("Executing Falcon Server Rolling Upgrade pre-restart") conf_select.select(params.stack_name, "falcon", params.version) hdp_select.select("falcon-server", params.version) falcon_server_upgrade.pre_start_restore()
def default(name, default_value): subdicts = filter(None, name.split('/')) curr_dict = Script.get_config() for x in subdicts: if x in curr_dict: curr_dict = curr_dict[x] else: if not isinstance(default_value, UnknownConfiguration): Logger.debug("Cannot find configuration: '%s'. Using '%s' value as default" % (name, default_value)) return default_value return curr_dict
def test_structured_out(self, open_mock): script = Script() script.stroutfile = '' self.assertEqual(Script.structuredOut, {}) script.put_structured_out({"1": "1"}) self.assertEqual(Script.structuredOut, {"1": "1"}) self.assertTrue(open_mock.called) script.put_structured_out({"2": "2"}) self.assertEqual(open_mock.call_count, 2) self.assertEqual(Script.structuredOut, {"1": "1", "2": "2"}) #Overriding script.put_structured_out({"1": "3"}) self.assertEqual(open_mock.call_count, 3) self.assertEqual(Script.structuredOut, {"1": "3", "2": "2"})
def pre_upgrade_restart(self, env, upgrade_type=None): Logger.info("Executing Metastore Stack Upgrade pre-restart") import params env.set_params(params) is_stack_hdp_23 = Script.is_hdp_stack_greater_or_equal("2.3") is_upgrade = params.upgrade_direction == Direction.UPGRADE if is_stack_hdp_23 and is_upgrade: self.upgrade_schema(env) if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: conf_select.select(params.stack_name, "hive", params.version) hdp_select.select("hive-metastore", params.version)
def test_install_packages(self, package_provider_mock): no_packages_config = { "hostLevelParams": { "repo_info": '[{"baseUrl":"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0","osType":"centos6","repoId":"HDP-2.0._","repoName":"HDP","defaultBaseUrl":"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0"}]' } } empty_config = { "hostLevelParams": { "package_list": "", "repo_info": '[{"baseUrl":"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0","osType":"centos6","repoId":"HDP-2.0._","repoName":"HDP","defaultBaseUrl":"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0"}]', } } dummy_config = { "hostLevelParams": { "package_list": '[{"type":"rpm","name":"hbase"},' '{"type":"rpm","name":"yet-another-package"}]', "repo_info": '[{"baseUrl":"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0","osType":"centos6","repoId":"HDP-2.0._","repoName":"HDP","defaultBaseUrl":"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0"}]', "service_repo_info": '[{"mirrorsList":"abc","osType":"centos6","repoId":"HDP-2.0._","repoName":"HDP","defaultBaseUrl":"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0"}]', } } # Testing config without any keys with Environment(".", test_mode=True) as env: script = Script() Script.config = no_packages_config script.install_packages(env) resource_dump = pprint.pformat(env.resource_list) self.assertEquals(resource_dump, "[]") # Testing empty package list with Environment(".", test_mode=True) as env: script = Script() Script.config = empty_config script.install_packages(env) resource_dump = pprint.pformat(env.resource_list) self.assertEquals(resource_dump, "[]") # Testing installing of a list of packages with Environment(".", test_mode=True) as env: Script.config = dummy_config script.install_packages("env") resource_dump = pprint.pformat(env.resource_list) self.assertEqual( resource_dump, "[Repository['HDP-2.0._'],\n Repository['HDP-2.0._'],\n Package['hbase'],\n Package['yet-another-package']]", )
def test_set_version(self, get_stack_to_component_mock): good_config = { 'hostLevelParams': { 'stack_name': "HDP", 'stack_version': "2.2" }, 'commandParams': { 'version': "2.2.0.0-2041" } } get_stack_to_component_mock.return_value = {"HDP": "kafka-broker"} # Testing default workflow with Environment(".", test_mode=True) as env: script = Script() Script.config = good_config script.set_version() resource_dump = pprint.pformat(env.resource_list) self.assertEquals(resource_dump, '[u"Execute[\'(\'/usr/bin/hdp-select\', \'set\', \'kafka-broker\', \'2.2.0.0-2041\')\']"]') # Component does not provide mapping get_stack_to_component_mock.return_value = {} with Environment(".", test_mode=True) as env: script = Script() Script.config = good_config script.set_version() resource_dump = pprint.pformat(env.resource_list) self.assertEquals(resource_dump, '[]') # Component provided mapping, but configuration is not complete (testing fallback) get_stack_to_component_mock.return_value = {"HDP": "kafka-broker"} bad_config = {} with Environment(".", test_mode=True) as env: script = Script() Script.config = bad_config script.set_version() resource_dump = pprint.pformat(env.resource_list) self.assertEquals(resource_dump, '[]')
def remove_hdp_21(self, env): """ During Express Upgrade from HDP 2.1 to any higher version (HDP 2.2 or 2.3), the HDP 2.1 bits must be uninstalled. This is because /usr/bin/hadoop used to be a shell script in HDP 2.1, but in HDP 2.3 it is a symlink to /usr/hdp/current/hadoop-client/bin/hadoop so both versions cannot coexist. """ Logger.info("Attempting to remove bits for HDP 2.1") config = Script.get_config() packages_to_remove = ["zookeeper", "hadoop", "hadoop-lzo", "hadoop-hdfs", "hadoop-libhdfs", "hadoop-yarn", "hadoop-client", "hadoop-mapreduce", "hive", "hive-hcatalog", "hive-jdbc", "hive-webhcat", "hcatalog", "webhcat-tar-hive", "webhcat-tar-pig", "oozie", "oozie-client", "pig", "sqoop", "tez" "falcon", "storm", "flume", "hbase", "phoenix"] packages_to_remove.reverse() Logger.info("Packages to remove: {0}".format(" ".join(packages_to_remove))) for name in packages_to_remove: Logger.info("Attempting to remove {0}".format(name)) Package(name, action="remove")
def link_configs(struct_out_file): """ Links configs, only on a fresh install of HDP-2.3 and higher """ if not Script.is_hdp_stack_greater_or_equal("2.3"): Logger.info("Can only link configs for HDP-2.3 and higher.") return json_version = load_version(struct_out_file) if not json_version: Logger.info("Could not load 'version' from {0}".format(struct_out_file)) return for k, v in conf_select.PACKAGE_DIRS.iteritems(): conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def get_not_managed_resources(): """ Returns a list of not managed hdfs paths. The result contains all paths from hostLevelParams/not_managed_hdfs_path_list except config values from cluster-env/managed_hdfs_resource_property_names """ config = Script.get_config() not_managed_hdfs_path_list = json.loads(config['hostLevelParams']['not_managed_hdfs_path_list'])[:] managed_hdfs_resource_property_names = config['configurations']['cluster-env']['managed_hdfs_resource_property_names'] managed_hdfs_resource_property_list = filter(None, [property.strip() for property in managed_hdfs_resource_property_names.split(',')]) for property_name in managed_hdfs_resource_property_list: property_value = default('/configurations/' + property_name, None) if property_value == None: Logger.warning(("Property {0} from cluster-env/managed_hdfs_resource_property_names not found in configurations. " "Management of this DFS resource will not be forced.").format(property_name)) else: while property_value in not_managed_hdfs_path_list: not_managed_hdfs_path_list.remove(property_value) return not_managed_hdfs_path_list
def expect(name, expected_type, default_value=None): """ Expect configuration to be of certain type. If it is not, give a reasonable error message to user. Optionally if the configuration is not found default_value for it can be returned. """ subdicts = filter(None, name.split('/')) curr_dict = Script.get_config() for x in subdicts: if x in curr_dict: curr_dict = curr_dict[x] else: if default_value: return default_value return UnknownConfiguration(curr_dict[-1]) value = curr_dict if expected_type == bool: if isinstance(value, bool): return value elif isinstance(value, basestring): if value != None and value.lower() == "true": value = True elif value != None and value.lower() == "false": value = False else: raise Fail("Configuration {0} expected to be boolean (true or false), but found '{1}'".format(name, value)) else: type_name = type(value).__name__ raise Fail("Configuration {0} expected to be boolean (true or false), but found instance of unknown type '{1}'".format(name, type_name)) elif expected_type in [int, long, float]: try: value = expected_type(value) except (ValueError, TypeError): raise Fail("Configuration {0} expected to be number, but found '{1}'".format(name, value)) return value
import functools import os from resource_management.libraries.functions import format from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions.default import default from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.script import Script from resource_management.libraries.functions.version import format_stack_version import status_params # server configurations config = Script.get_config() tmp_dir = Script.get_tmp_dir() install_dir = config['configurations']['metron-env']['install_dir'] download_url = config['configurations']['metron-env']['download_url'] filename = download_url.split('/')[-1] version_dir = filename.replace('.tar.gz', '').replace('.tgz', '') hostname = config['hostname'] metron_home = status_params.metron_home metron_version = config['configurations']['metron-env']['metron_version'] parsers = status_params.parsers parser_error_topic = config['configurations']['metron-parsers-env'][ 'parser_error_topic'] geoip_hdfs_dir = "/apps/metron/geo/default/"
def setup_ranger_admin(upgrade_type=None): import params if upgrade_type is None: upgrade_type = Script.get_upgrade_type( default("/commandParams/upgrade_type", "")) ranger_home = params.ranger_home ranger_conf = params.ranger_conf Directory(ranger_conf, owner=params.unix_user, group=params.unix_group, create_parents=True) copy_jdbc_connector(ranger_home) File( format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content=DownloadSource( format("{jdk_location}/{check_db_connection_jar_name}")), mode=0644, ) cp = format("{check_db_connection_jar}") if params.db_flavor.lower() == 'sqla': cp = cp + os.pathsep + format("{ranger_home}/ews/lib/sajdbc4.jar") else: cp = cp + os.pathsep + format("{driver_curl_target}") cp = cp + os.pathsep + format("{ranger_home}/ews/lib/*") db_connection_check_command = format( "{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_jdbc_connection_url}' {ranger_db_user} {ranger_db_password!p} {ranger_jdbc_driver}" ) env_dict = {} if params.db_flavor.lower() == 'sqla': env_dict = {'LD_LIBRARY_PATH': params.ld_lib_path} Execute(db_connection_check_command, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10, environment=env_dict) Execute( ('ln', '-sf', format('{ranger_home}/ews/webapp/WEB-INF/classes/conf'), format('{ranger_home}/conf')), not_if=format("ls {ranger_home}/conf"), only_if=format("ls {ranger_home}/ews/webapp/WEB-INF/classes/conf"), sudo=True) if upgrade_type is not None: src_file = format( '{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/ranger-admin-default-site.xml' ) dst_file = format('{ranger_home}/conf/ranger-admin-default-site.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) src_file = format( '{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/security-applicationContext.xml' ) dst_file = format('{ranger_home}/conf/security-applicationContext.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) Directory( format('{ranger_home}/'), owner=params.unix_user, group=params.unix_group, recursive_ownership=True, ) Directory(params.ranger_pid_dir, mode=0755, owner=params.unix_user, group=params.user_group, cd_access="a", create_parents=True) if params.stack_supports_pid: File( format('{ranger_conf}/ranger-admin-env-piddir.sh'), content=format( "export RANGER_PID_DIR_PATH={ranger_pid_dir}\nexport RANGER_USER={unix_user}" ), owner=params.unix_user, group=params.unix_group, mode=0755) Directory(params.admin_log_dir, owner=params.unix_user, group=params.unix_group, create_parents=True, cd_access='a', mode=0755) File(format('{ranger_conf}/ranger-admin-env-logdir.sh'), content=format("export RANGER_ADMIN_LOG_DIR={admin_log_dir}"), owner=params.unix_user, group=params.unix_group, mode=0755) if os.path.isfile(params.ranger_admin_default_file): File(params.ranger_admin_default_file, owner=params.unix_user, group=params.unix_group) else: Logger.warning( 'Required file {0} does not exist, copying the file to {1} path'. format(params.ranger_admin_default_file, ranger_conf)) src_file = format( '{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/ranger-admin-default-site.xml' ) dst_file = format('{ranger_home}/conf/ranger-admin-default-site.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) File(params.ranger_admin_default_file, owner=params.unix_user, group=params.unix_group) if os.path.isfile(params.security_app_context_file): File(params.security_app_context_file, owner=params.unix_user, group=params.unix_group) else: Logger.warning( 'Required file {0} does not exist, copying the file to {1} path'. format(params.security_app_context_file, ranger_conf)) src_file = format( '{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/security-applicationContext.xml' ) dst_file = format('{ranger_home}/conf/security-applicationContext.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) File(params.security_app_context_file, owner=params.unix_user, group=params.unix_group) if upgrade_type is not None and params.stack_supports_config_versioning: if os.path.islink('/usr/bin/ranger-admin'): Link('/usr/bin/ranger-admin', action="delete") Link('/usr/bin/ranger-admin', to=format('{ranger_home}/ews/ranger-admin-services.sh')) if default( "/configurations/ranger-admin-site/ranger.authentication.method", "") == 'PAM': d = '/etc/pam.d' if os.path.isdir(d): if os.path.isfile(os.path.join(d, 'ranger-admin')): Logger.info('ranger-admin PAM file already exists.') else: File(format('{d}/ranger-admin'), content=Template('ranger_admin_pam.j2'), owner=params.unix_user, group=params.unix_group, mode=0644) if os.path.isfile(os.path.join(d, 'ranger-remote')): Logger.info('ranger-remote PAM file already exists.') else: File(format('{d}/ranger-remote'), content=Template('ranger_remote_pam.j2'), owner=params.unix_user, group=params.unix_group, mode=0644) else: Logger.error( "Unable to use PAM authentication, /etc/pam.d/ directory does not exist." ) Execute(('ln', '-sf', format('{ranger_home}/ews/ranger-admin-services.sh'), '/usr/bin/ranger-admin'), not_if=format("ls /usr/bin/ranger-admin"), only_if=format("ls {ranger_home}/ews/ranger-admin-services.sh"), sudo=True) # remove plain-text password from xml configs ranger_admin_site_copy = {} ranger_admin_site_copy.update( params.config['configurations']['ranger-admin-site']) for prop in params.ranger_admin_password_properties: if prop in ranger_admin_site_copy: ranger_admin_site_copy[prop] = "_" XmlConfig("ranger-admin-site.xml", conf_dir=ranger_conf, configurations=ranger_admin_site_copy, configuration_attributes=params.config['configurationAttributes'] ['ranger-admin-site'], owner=params.unix_user, group=params.unix_group, mode=0644) Directory( os.path.join(ranger_conf, 'ranger_jaas'), mode=0700, owner=params.unix_user, group=params.unix_group, ) if params.stack_supports_ranger_log4j: File(format('{ranger_home}/ews/webapp/WEB-INF/log4j.properties'), owner=params.unix_user, group=params.unix_group, content=InlineTemplate(params.admin_log4j), mode=0644) do_keystore_setup(upgrade_type=upgrade_type) create_core_site_xml(ranger_conf) if params.stack_supports_ranger_kerberos and params.security_enabled: if params.is_hbase_ha_enabled and params.ranger_hbase_plugin_enabled: XmlConfig( "hbase-site.xml", conf_dir=ranger_conf, configurations=params.config['configurations']['hbase-site'], configuration_attributes=params. config['configurationAttributes']['hbase-site'], owner=params.unix_user, group=params.unix_group, mode=0644) if params.is_namenode_ha_enabled and params.ranger_hdfs_plugin_enabled: XmlConfig( "hdfs-site.xml", conf_dir=ranger_conf, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params. config['configurationAttributes']['hdfs-site'], owner=params.unix_user, group=params.unix_group, mode=0644)
regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ from resource_management.libraries.script import Script from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions import default, format config = Script.get_config() tmp_dir = Script.get_tmp_dir() #script check_status_script = "{0}/checkStatus.sh".format(tmp_dir) #process name proc_nimbus_name = "com.tencent.jstorm.daemon.nimbus.NimbusServer" proc_supervisor_name = "com.tencent.jstorm.daemon.supervisor.Supervisor" proc_ui_server_name = "com.tencent.jstorm.ui.core.UIServer"
from resource_management.libraries.functions.is_empty import is_empty from resource_management.libraries.functions.constants import Direction from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.stack_features import get_stack_feature_version from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.get_bare_principal import get_bare_principal # a map of the Ambari role to the component name # for use with <stack-root>/current/<component> SERVER_ROLE_DIRECTORY_MAP = { 'RANGER_ADMIN': 'ranger-admin', 'RANGER_USERSYNC': 'ranger-usersync', 'RANGER_TAGSYNC': 'ranger-tagsync' } component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "RANGER_ADMIN") config = Script.get_config() tmp_dir = Script.get_tmp_dir() stack_root = Script.get_stack_root() stack_name = default("/hostLevelParams/stack_name", None) version = default("/commandParams/version", None) stack_version_unformatted = config['hostLevelParams']['stack_version'] stack_version_formatted = format_stack_version(stack_version_unformatted) major_stack_version = get_major_version(stack_version_formatted) upgrade_marker_file = format("{tmp_dir}/rangeradmin_ru.inprogress") xml_configurations_supported = config['configurations']['ranger-env'][
#!/usr/bin/env python from resource_management.libraries.script import Script from resource_management.libraries.functions import default import functools from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.resources import HdfsResource from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources # server configurations config = Script.get_config() stack_root = Script.get_stack_root() wormhole_user = config['configurations']['wormhole-env']['wormhole_user'] log_dir = config['configurations']['wormhole-env']['log_dir'] pid_dir = config['configurations']['wormhole-env']['pid_dir'] hostname = config['agentLevelParams']['hostname'] java64_home = config['ambariLevelParams']['java_home'] install_dir = stack_root + '/wormhole' download_url = config['configurations']['wormhole-env']['download_url'] filename = download_url.split('/')[-1] version_dir = filename.replace('.tar.gz', '').replace('.tgz', '').replace('.tar.bz2', '') conf_content = config['configurations']['wormhole-env']['conf_content'] cluster_id = config['configurations']['wormhole-env']['cluster_id'] db_url = config['configurations']['wormhole-env']['db_url'] cluster_name = config['clusterName'] flink_home = stack_root + '/flink'
""" """ from resource_management.libraries.script import Script from resource_management.libraries.functions import format app = 'unloadCron' app_exec = format('{app}.pl') app_root = '/var/lib/unload' app_sbin = format('{app_root}/sbin') app_bin = format('{app_root}/bin') app_start = 'unload_start.sh' config = Script.get_config() base_dir = config['configurations']['unload-env']['base.dir'] user = config['configurations']['unload-env']['unload_user'] group = config['configurations']['unload-env']['user_group'] concurrency = config['configurations']['unload-site']['concurrency'] throttle = config['configurations']['unload-site']['throttle'] garbage_seconds = config['configurations']['unload-site']['garbage.seconds']
def actionexecute(self, env): resolve_ambari_config() # Parse parameters from command json file. config = Script.get_config() host_name = socket.gethostname() version = default('/roleParams/version', None) # These 2 variables are optional service_package_folder = default('/roleParams/service_package_folder', None) hooks_folder = default('/roleParams/hooks_folder', None) tasks = json.loads(config['roleParams']['tasks']) if tasks: for t in tasks: task = ExecuteTask(t) Logger.info(str(task)) # If a (script, function) exists, it overwrites the command. if task.script and task.function: file_cache = FileCache(agent_config) server_url_prefix = default( '/hostLevelParams/jdk_location', "") if service_package_folder and hooks_folder: command_paths = { "commandParams": { "service_package_folder": service_package_folder, "hooks_folder": hooks_folder } } base_dir = file_cache.get_service_base_dir( command_paths, server_url_prefix) else: base_dir = file_cache.get_custom_actions_base_dir( server_url_prefix) script_path = os.path.join(base_dir, task.script) if not os.path.exists(script_path): message = "Script %s does not exist" % str(script_path) raise Fail(message) # Notice that the script_path is now the fully qualified path, and the # same command-#.json file is used. # Also, the python wrapper is used, since it sets up the correct environment variables command_params = [ "/usr/bin/ambari-python-wrap", script_path, task.function, self.command_data_file, self.basedir, self.stroutfile, self.logging_level, Script.get_tmp_dir() ] task.command = "source /var/lib/ambari-agent/ambari-env.sh ; " + " ".join( command_params) # Replace redundant whitespace to make the unit tests easier to validate task.command = re.sub("\s+", " ", task.command).strip() if task.command: task.command = replace_variables(task.command, host_name, version) shell.checked_call(task.command, logoutput=True, quiet=True)
See the License for the specific language governing permissions and limitations under the License. """ import os from ambari_commons.constants import AMBARI_SUDO_BINARY from resource_management.libraries.script import Script from resource_management.libraries.functions import default from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import hdp_select from resource_management.libraries.functions import format_jvm_option from resource_management.libraries.functions.version import format_hdp_stack_version config = Script.get_config() tmp_dir = Script.get_tmp_dir() dfs_type = default("/commandParams/dfs_type", "") is_parallel_execution_enabled = int( default("/agentConfigParams/agent/parallel_execution", 0)) == 1 sudo = AMBARI_SUDO_BINARY stack_version_unformatted = str(config['hostLevelParams']['stack_version']) hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) # current host stack version current_version = default("/hostLevelParams/current_version", None)
#!/usr/bin/env python from resource_management.libraries.script import Script config = Script.get_config() airflow_user = "******" airflow_group = "airflow" airflow_home = config['configurations']['airflow-core-site']['airflow_home'] dirs = [ airflow_home, config['configurations']['airflow-core-site']['dags_folder'], config['configurations']['airflow-core-site']['base_log_folder'], config['configurations']['airflow-core-site']['plugins_folder'], config['configurations']['airflow-scheduler-site'] ['child_process_log_directory'] ]
from resource_management.libraries.script import Script from resource_management.libraries.functions.version import format_stack_version, get_major_version from resource_management.libraries.functions.format import format from resource_management.libraries.functions.default import default from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.stack_features import get_stack_feature_version from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.get_bare_principal import get_bare_principal from resource_management.libraries.functions.is_empty import is_empty from resource_management.libraries.functions.setup_ranger_plugin_xml import generate_ranger_service_config from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import get_kinit_path from resource_management.core.exceptions import Fail config = Script.get_config() tmp_dir = Script.get_tmp_dir() stack_root = Script.get_stack_root() stack_name = default("/clusterLevelParams/stack_name", None) version = default("/commandParams/version", None) upgrade_direction = default("/commandParams/upgrade_direction", None) stack_version_unformatted = config['clusterLevelParams']['stack_version'] stack_version_formatted = format_stack_version(stack_version_unformatted) major_stack_version = get_major_version(stack_version_formatted) # get the correct version to use for checking stack features version_for_stack_feature_checks = get_stack_feature_version(config) stack_supports_config_versioning = check_stack_feature(
from resource_management.libraries.functions.format import format from resource_management.libraries.functions.default import default from resource_management.libraries.functions.is_empty import is_empty from resource_management.libraries.functions.constants import Direction from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions import StackFeature # a map of the Ambari role to the component name # for use with <stack-root>/current/<component> SERVER_ROLE_DIRECTORY_MAP = { 'RANGER_ADMIN' : 'ranger-admin', 'RANGER_USERSYNC' : 'ranger-usersync', 'RANGER_TAGSYNC' : 'ranger-tagsync' } component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "RANGER_ADMIN") config = Script.get_config() tmp_dir = Script.get_tmp_dir() stack_root = Script.get_stack_root() stack_name = default("/hostLevelParams/stack_name", None) version = default("/commandParams/version", None) host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) stack_version_unformatted = config['hostLevelParams']['stack_version'] stack_version_formatted = format_stack_version(stack_version_unformatted) upgrade_marker_file = format("{tmp_dir}/rangeradmin_ru.inprogress") xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported']
limitations under the License. """ from ambari_commons.constants import AMBARI_SUDO_BINARY from resource_management.libraries.script import Script from resource_management.libraries.functions import default from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import hdp_select from resource_management.libraries.functions import format_jvm_option from resource_management.libraries.functions.version import format_hdp_stack_version from resource_management.core.system import System from ambari_commons.os_check import OSCheck config = Script.get_config() sudo = AMBARI_SUDO_BINARY stack_version_unformatted = str(config['hostLevelParams']['stack_version']) hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) # default hadoop params mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*" hadoop_libexec_dir = hdp_select.get_hadoop_dir("libexec") hadoop_conf_empty_dir = "/etc/hadoop/conf.empty" # HDP 2.2+ params if Script.is_hdp_stack_greater_or_equal("2.2"): mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*" # not supported in HDP 2.2+
from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import format from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions import stack_select from resource_management.libraries.functions.default import default from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.script import Script from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions import StackFeature import status_params # server configurations config = Script.get_config() tmp_dir = Script.get_tmp_dir() hdp_version = default("/commandParams/version", None) hostname = config['hostname'] metron_home = status_params.metron_home metron_apps_hdfs_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir'] parsers = status_params.parsers parser_error_topic = config['configurations']['metron-parsers-env']['parser_error_topic'] geoip_hdfs_dir = metron_apps_hdfs_dir + "/geo/default/" asn_hdfs_dir = metron_apps_hdfs_dir + "/asn/default/" hbase_coprocessor_local_dir = format("{metron_home}/coprocessor") hbase_coprocessor_hdfs_dir = metron_apps_hdfs_dir + "/coprocessor" metron_user = status_params.metron_user
limitations under the License. """ from resource_management.libraries.functions.version import format_hdp_stack_version from resource_management.libraries.functions.default import default from resource_management.libraries.functions.get_kinit_path import get_kinit_path from resource_management.libraries.script import Script # a map of the Ambari role to the component name # for use with /usr/hdp/current/<component> SERVER_ROLE_DIRECTORY_MAP = { 'SQOOP' : 'sqoop-client' } component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "SQOOP") config = Script.get_config() ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] stack_name = default("/hostLevelParams/stack_name", None) stack_version_unformatted = str(config['hostLevelParams']['stack_version']) hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) # New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade version = default("/commandParams/version", None) # default hadoop params sqoop_conf_dir = "/usr/lib/sqoop/conf" sqoop_lib = "/usr/lib/sqoop/lib"
def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=True, timeout=None, user=None): # TODO implement timeout, wait_for_finish Logger.info("Executing %s" % (command)) if user: domain, username = UserHelper.parse_user_name(user, ".") proc_token = OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES) old_states = [] privileges = [ SE_ASSIGNPRIMARYTOKEN_NAME, SE_INCREASE_QUOTA_NAME, ] for priv in privileges: old_states.append(QueryPrivilegeState(proc_token, priv)) AdjustPrivilege(proc_token, priv) QueryPrivilegeState(proc_token, priv) user_token = LogonUser(username, domain, Script.get_password(user), win32con.LOGON32_LOGON_SERVICE, win32con.LOGON32_PROVIDER_DEFAULT) env_token = DuplicateTokenEx(user_token, SecurityIdentification, TOKEN_QUERY, TokenPrimary) # getting updated environment for impersonated user and merge it with custom env current_env = CreateEnvironmentBlock(env_token, False) current_env = _merge_env(current_env, env) si = STARTUPINFO() out_handle, err_handle, out_file, err_file = _create_tmp_files(current_env) ok, si.hStdInput = _safe_duplicate_handle(GetStdHandle(STD_INPUT_HANDLE)) if not ok: raise Exception("Unable to create StdInput for child process") ok, si.hStdOutput = _safe_duplicate_handle(out_handle) if not ok: raise Exception("Unable to create StdOut for child process") ok, si.hStdError = _safe_duplicate_handle(err_handle) if not ok: raise Exception("Unable to create StdErr for child process") Logger.debug("Redirecting stdout to '{0}', stderr to '{1}'".format(out_file.name, err_file.name)) si.dwFlags = win32con.STARTF_USESTDHANDLES si.lpDesktop = "" try: info = CreateProcessAsUser(user_token, None, command, None, None, 1, win32con.CREATE_NO_WINDOW, current_env, cwd, si) hProcess, hThread, dwProcessId, dwThreadId = info hThread.Close() try: WaitForSingleObject(hProcess, INFINITE) except KeyboardInterrupt: pass out, err = _get_files_output(out_file, err_file) code = GetExitCodeProcess(hProcess) finally: for priv in privileges: old_state = old_states.pop(0) AdjustPrivilege(proc_token, priv, old_state) else: # getting updated environment for current process and merge it with custom env cur_token = OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY) current_env = CreateEnvironmentBlock(cur_token, False) current_env = _merge_env(current_env, env) proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd, env=current_env, shell=False) out, err = proc.communicate() code = proc.returncode if logoutput and out: Logger.info(out) if logoutput and err: Logger.info(err) return code, out, err
import status_params from ambari_commons.constants import AMBARI_SUDO_BINARY from resource_management.libraries.functions.constants import Direction from resource_management.libraries.functions import format from resource_management.libraries.functions.version import format_hdp_stack_version from resource_management.libraries.functions.default import default from resource_management.libraries.functions.get_bare_principal import get_bare_principal from resource_management.libraries.script import Script from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions import hdp_select from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import get_kinit_path # server configurations config = Script.get_config() tmp_dir = Script.get_tmp_dir() sudo = AMBARI_SUDO_BINARY stack_name = default("/hostLevelParams/stack_name", None) upgrade_direction = default("/commandParams/upgrade_direction", Direction.UPGRADE) version = default("/commandParams/version", None) storm_component_home_dir = status_params.storm_component_home_dir conf_dir = status_params.conf_dir stack_version_unformatted = str(config['hostLevelParams']['stack_version']) hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) stack_is_hdp22_or_further = Script.is_hdp_stack_greater_or_equal("2.2") # default hadoop params
import re import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. from resource_management.libraries.script import Script from resource_management.libraries.functions import default from resource_management.libraries.functions import format from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import hdp_select from resource_management.libraries.functions import format_jvm_option from resource_management.libraries.functions.version import format_hdp_stack_version from resource_management.libraries.functions.version import compare_versions from ambari_commons.os_check import OSCheck config = Script.get_config() tmp_dir = Script.get_tmp_dir() artifact_dir = format("{tmp_dir}/AMBARI-artifacts/") jce_policy_zip = default("/hostLevelParams/jce_name", None) # None when jdk is already installed by user jce_location = config['hostLevelParams']['jdk_location'] jdk_name = default("/hostLevelParams/jdk_name", None) java_home = config['hostLevelParams']['java_home'] java_version = int(config['hostLevelParams']['java_version']) ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] stack_version_unformatted = str(config['hostLevelParams']['stack_version']) hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) security_enabled = config['configurations']['cluster-env']['security_enabled']
from resource_management.libraries.script import Script from resource_management.libraries.functions import default from resource_management.libraries.functions import format from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import format_jvm_option from resource_management.libraries.functions.is_empty import is_empty from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.expect import expect from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.stack_features import get_stack_feature_version from resource_management.libraries.functions.get_architecture import get_architecture from ambari_commons.constants import AMBARI_SUDO_BINARY config = Script.get_config() tmp_dir = Script.get_tmp_dir() stack_root = Script.get_stack_root() architecture = get_architecture() dfs_type = default("/commandParams/dfs_type", "") artifact_dir = format("{tmp_dir}/AMBARI-artifacts/") jdk_name = default("/ambariLevelParams/jdk_name", None) java_home = config['ambariLevelParams']['java_home'] java_version = expect("/ambariLevelParams/java_version", int) jdk_location = config['ambariLevelParams']['jdk_location'] hadoop_custom_extensions_enabled = default(