def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: expectations = {} expectations.update(build_expectations( 'krb5JAASLogin', None, ['keytab', 'principal'], None )) expectations.update(build_expectations( 'gateway-site', { "gateway.hadoop.kerberos.secured" : "true" }, None, None )) security_params = { "krb5JAASLogin": { 'keytab': status_params.knox_keytab_path, 'principal': status_params.knox_principal_name } } security_params.update(get_params_from_filesystem(status_params.knox_conf_dir, {"gateway-site.xml" : FILE_TYPE_XML})) result_issues = validate_security_config_properties(security_params, expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ( 'krb5JAASLogin' not in security_params or 'keytab' not in security_params['krb5JAASLogin'] or 'principal' not in security_params['krb5JAASLogin']): self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out({"securityIssuesFound": "Keytab file and principal are not set."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.knox_user, security_params['krb5JAASLogin']['keytab'], security_params['krb5JAASLogin']['principal'], status_params.hostname, status_params.temp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: props_value_check = {"yarn.timeline-service.enabled": "true", "yarn.timeline-service.http-authentication.type": "kerberos", "yarn.acl.enable": "true"} props_empty_check = ["yarn.timeline-service.principal", "yarn.timeline-service.keytab", "yarn.timeline-service.http-authentication.kerberos.principal", "yarn.timeline-service.http-authentication.kerberos.keytab"] props_read_check = ["yarn.timeline-service.keytab", "yarn.timeline-service.http-authentication.kerberos.keytab"] yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check, props_read_check) yarn_expectations ={} yarn_expectations.update(yarn_site_props) security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, {'yarn-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties(security_params, yarn_expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ( 'yarn-site' not in security_params or 'yarn.timeline-service.keytab' not in security_params['yarn-site'] or 'yarn.timeline-service.principal' not in security_params['yarn-site']) \ or 'yarn.timeline-service.http-authentication.kerberos.keytab' not in security_params['yarn-site'] \ or 'yarn.timeline-service.http-authentication.kerberos.principal' not in security_params['yarn-site']: self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out( {"securityIssuesFound": "Keytab file or principal are not set property."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.yarn_user, security_params['yarn-site']['yarn.timeline-service.keytab'], security_params['yarn-site']['yarn.timeline-service.principal'], status_params.hostname, status_params.tmp_dir) cached_kinit_executor(status_params.kinit_path_local, status_params.yarn_user, security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.keytab'], security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.principal'], status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: expectations = {} expectations.update(build_expectations( 'krb5JAASLogin', None, ['keytab', 'principal'], None )) expectations.update(build_expectations( 'gateway-site', { "gateway.hadoop.kerberos.secured" : "true" }, None, None )) security_params = { "krb5JAASLogin": { 'keytab': status_params.knox_keytab_path, 'principal': status_params.knox_principal_name } } security_params.update(get_params_from_filesystem(status_params.knox_conf_dir, {"gateway-site.xml" : FILE_TYPE_XML})) result_issues = validate_security_config_properties(security_params, expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ( 'krb5JAASLogin' not in security_params or 'keytab' not in security_params['krb5JAASLogin'] or 'principal' not in security_params['krb5JAASLogin']): self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out({"securityIssuesFound": "Keytab file and principal are not set."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.knox_user, security_params['krb5JAASLogin']['keytab'], security_params['krb5JAASLogin']['principal'], status_params.hostname, status_params.temp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) props_value_check = {"hbase.security.authentication": "kerberos", "hbase.security.authorization": "true"} props_empty_check = ["hbase.zookeeper.property.authProvider.1", "hbase.master.keytab.file", "hbase.master.kerberos.principal", "hbase.regionserver.keytab.file", "hbase.regionserver.kerberos.principal" ] props_read_check = ['hbase.master.keytab.file', 'hbase.regionserver.keytab.file'] ams_hbase_site_expectations = build_expectations('hbase-site', props_value_check, props_empty_check, props_read_check) expectations = {} expectations.update(ams_hbase_site_expectations) security_params = get_params_from_filesystem(status_params.ams_hbase_conf_dir, {'hbase-site.xml': FILE_TYPE_XML}) is_hbase_distributed = security_params['hbase-site']['hbase.cluster.distributed'] # for embedded mode, when HBase is backed by file, security state is SECURED_KERBEROS by definition when cluster is secured if status_params.security_enabled and not is_hbase_distributed: self.put_structured_out({"securityState": "SECURED_KERBEROS"}) return result_issues = validate_security_config_properties(security_params, expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ('hbase-site' not in security_params or 'hbase.master.keytab.file' not in security_params['hbase-site'] or 'hbase.master.kerberos.principal' not in security_params['hbase-site']): self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out( {"securityIssuesFound": "Keytab file or principal are not set property."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.hbase_user, security_params['hbase-site']['hbase.master.keytab.file'], security_params['hbase-site']['hbase.master.kerberos.principal'], status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % ( cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: props_value_check = {"yarn.timeline-service.enabled": "true", "yarn.timeline-service.http-authentication.type": "kerberos", "yarn.acl.enable": "true"} props_empty_check = ["yarn.timeline-service.principal", "yarn.timeline-service.keytab", "yarn.timeline-service.http-authentication.kerberos.principal", "yarn.timeline-service.http-authentication.kerberos.keytab"] props_read_check = ["yarn.timeline-service.keytab", "yarn.timeline-service.http-authentication.kerberos.keytab"] yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check, props_read_check) yarn_expectations ={} yarn_expectations.update(yarn_site_props) security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, {'yarn-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties(security_params, yarn_expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ( 'yarn-site' not in security_params or 'yarn.timeline-service.keytab' not in security_params['yarn-site'] or 'yarn.timeline-service.principal' not in security_params['yarn-site']) \ or 'yarn.timeline-service.http-authentication.kerberos.keytab' not in security_params['yarn-site'] \ or 'yarn.timeline-service.http-authentication.kerberos.principal' not in security_params['yarn-site']: self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out( {"securityIssuesFound": "Keytab file or principal are not set property."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.yarn_user, security_params['yarn-site']['yarn.timeline-service.keytab'], security_params['yarn-site']['yarn.timeline-service.principal'], status_params.hostname, status_params.tmp_dir) cached_kinit_executor(status_params.kinit_path_local, status_params.yarn_user, security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.keytab'], security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.principal'], status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) props_value_check = {"hbase.security.authentication": "kerberos", "hbase.security.authorization": "true"} props_empty_check = ["hbase.zookeeper.property.authProvider.1", "hbase.master.keytab.file", "hbase.master.kerberos.principal", "hbase.regionserver.keytab.file", "hbase.regionserver.kerberos.principal" ] props_read_check = ['hbase.master.keytab.file', 'hbase.regionserver.keytab.file'] ams_hbase_site_expectations = build_expectations('hbase-site', props_value_check, props_empty_check, props_read_check) expectations = {} expectations.update(ams_hbase_site_expectations) security_params = get_params_from_filesystem(status_params.ams_hbase_conf_dir, {'hbase-site.xml': FILE_TYPE_XML}) is_hbase_distributed = security_params['hbase-site']['hbase.cluster.distributed'] # for embedded mode, when HBase is backed by file, security state is SECURED_KERBEROS by definition when cluster is secured if status_params.security_enabled and not is_hbase_distributed: self.put_structured_out({"securityState": "SECURED_KERBEROS"}) return result_issues = validate_security_config_properties(security_params, expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ('hbase-site' not in security_params or 'hbase.master.keytab.file' not in security_params['hbase-site'] or 'hbase.master.kerberos.principal' not in security_params['hbase-site']): self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out( {"securityIssuesFound": "Keytab file or principal are not set property."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.hbase_user, security_params['hbase-site']['hbase.master.keytab.file'], security_params['hbase-site']['hbase.master.kerberos.principal'], status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % ( cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: props_value_check = {"hive.server2.authentication": "KERBEROS", "hive.metastore.sasl.enabled": "true", "hive.security.authorization.enabled": "true"} props_empty_check = ["hive.server2.authentication.kerberos.keytab", "hive.server2.authentication.kerberos.principal", "hive.server2.authentication.spnego.principal", "hive.server2.authentication.spnego.keytab"] props_read_check = ["hive.server2.authentication.kerberos.keytab", "hive.server2.authentication.spnego.keytab"] hive_site_props = build_expectations('hive-site', props_value_check, props_empty_check, props_read_check) hive_expectations ={} hive_expectations.update(hive_site_props) security_params = get_params_from_filesystem(status_params.hive_server_interactive_conf_dir, {'hive-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties(security_params, hive_expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if 'hive-site' not in security_params \ or 'hive.server2.authentication.kerberos.keytab' not in security_params['hive-site'] \ or 'hive.server2.authentication.kerberos.principal' not in security_params['hive-site'] \ or 'hive.server2.authentication.spnego.keytab' not in security_params['hive-site'] \ or 'hive.server2.authentication.spnego.principal' not in security_params['hive-site']: self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.hive_user, security_params['hive-site']['hive.server2.authentication.kerberos.keytab'], security_params['hive-site']['hive.server2.authentication.kerberos.principal'], status_params.hostname, status_params.tmp_dir) cached_kinit_executor(status_params.kinit_path_local, status_params.hive_user, security_params['hive-site']['hive.server2.authentication.spnego.keytab'], security_params['hive-site']['hive.server2.authentication.spnego.principal'], status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: # Expect the following files to be available in status_params.config_dir: # storm_jaas.conf try: props_value_check = None props_empty_check = [ 'StormServer/keyTab', 'StormServer/principal' ] props_read_check = ['StormServer/keyTab'] storm_env_expectations = build_expectations( 'storm_jaas', props_value_check, props_empty_check, props_read_check) storm_expectations = {} storm_expectations.update(storm_env_expectations) security_params = get_params_from_filesystem( status_params.conf_dir, {'storm_jaas.conf': FILE_TYPE_JAAS_CONF}) result_issues = validate_security_config_properties( security_params, storm_expectations) if not result_issues: # If all validations passed successfully # Double check the dict before calling execute if ('storm_jaas' not in security_params or 'StormServer' not in security_params['storm_jaas'] or 'keyTab' not in security_params['storm_jaas']['StormServer'] or 'principal' not in security_params['storm_jaas'] ['StormServer']): self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({ "securityIssuesFound": "Keytab file or principal are not set property." }) return cached_kinit_executor( status_params.kinit_path_local, status_params.storm_user, security_params['storm_jaas']['StormServer']['keyTab'], security_params['storm_jaas']['StormServer'] ['principal'], status_params.hostname, status_params.tmp_dir) self.put_structured_out( {"securityState": "SECURED_KERBEROS"}) else: issues = [] for cf in result_issues: issues.append( "Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out( {"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) props_value_check = {"hadoop.security.authentication": "kerberos", "hadoop.security.authorization": "true"} props_empty_check = ["hadoop.security.auth_to_local"] props_read_check = None core_site_expectations = build_expectations('core-site', props_value_check, props_empty_check, props_read_check) props_value_check = None props_empty_check = ['dfs.secondary.namenode.kerberos.internal.spnego.principal', 'dfs.secondary.namenode.keytab.file', 'dfs.secondary.namenode.kerberos.principal'] props_read_check = ['dfs.secondary.namenode.keytab.file'] hdfs_site_expectations = build_expectations('hdfs-site', props_value_check, props_empty_check, props_read_check) hdfs_expectations = {} hdfs_expectations.update(core_site_expectations) hdfs_expectations.update(hdfs_site_expectations) security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, {'core-site.xml': FILE_TYPE_XML, 'hdfs-site.xml': FILE_TYPE_XML}) if 'core-site' in security_params and 'hadoop.security.authentication' in security_params['core-site'] and \ security_params['core-site']['hadoop.security.authentication'].lower() == 'kerberos': result_issues = validate_security_config_properties(security_params, hdfs_expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ('hdfs-site' not in security_params or 'dfs.secondary.namenode.keytab.file' not in security_params['hdfs-site'] or 'dfs.secondary.namenode.kerberos.principal' not in security_params['hdfs-site']): self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out( {"securityIssuesFound": "Keytab file or principal are not set property."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.hdfs_user, security_params['hdfs-site']['dfs.secondary.namenode.keytab.file'], security_params['hdfs-site'][ 'dfs.secondary.namenode.kerberos.principal'], status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: props_value_check = {"*.falcon.authentication.type": "kerberos", "*.falcon.http.authentication.type": "kerberos"} props_empty_check = ["*.falcon.service.authentication.kerberos.principal", "*.falcon.service.authentication.kerberos.keytab", "*.falcon.http.authentication.kerberos.principal", "*.falcon.http.authentication.kerberos.keytab"] props_read_check = ["*.falcon.service.authentication.kerberos.keytab", "*.falcon.http.authentication.kerberos.keytab"] falcon_startup_props = build_expectations('startup', props_value_check, props_empty_check, props_read_check) falcon_expectations ={} falcon_expectations.update(falcon_startup_props) security_params = get_params_from_filesystem('/etc/falcon/conf', {'startup.properties': FILE_TYPE_PROPERTIES}) result_issues = validate_security_config_properties(security_params, falcon_expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ( 'startup' not in security_params or '*.falcon.service.authentication.kerberos.keytab' not in security_params['startup'] or '*.falcon.service.authentication.kerberos.principal' not in security_params['startup']) \ or '*.falcon.http.authentication.kerberos.keytab' not in security_params['startup'] \ or '*.falcon.http.authentication.kerberos.principal' not in security_params['startup']: self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out( {"securityIssuesFound": "Keytab file or principal are not set property."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.falcon_user, security_params['startup']['*.falcon.service.authentication.kerberos.keytab'], security_params['startup']['*.falcon.service.authentication.kerberos.principal'], status_params.hostname, status_params.tmp_dir) cached_kinit_executor(status_params.kinit_path_local, status_params.falcon_user, security_params['startup']['*.falcon.http.authentication.kerberos.keytab'], security_params['startup']['*.falcon.http.authentication.kerberos.principal'], status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: props_value_check = {"*.falcon.authentication.type": "kerberos", "*.falcon.http.authentication.type": "kerberos"} props_empty_check = ["*.falcon.service.authentication.kerberos.principal", "*.falcon.service.authentication.kerberos.keytab", "*.falcon.http.authentication.kerberos.principal", "*.falcon.http.authentication.kerberos.keytab"] props_read_check = ["*.falcon.service.authentication.kerberos.keytab", "*.falcon.http.authentication.kerberos.keytab"] falcon_startup_props = build_expectations('startup', props_value_check, props_empty_check, props_read_check) falcon_expectations ={} falcon_expectations.update(falcon_startup_props) security_params = get_params_from_filesystem('/etc/falcon/conf', {'startup.properties': FILE_TYPE_PROPERTIES}) result_issues = validate_security_config_properties(security_params, falcon_expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ( 'startup' not in security_params or '*.falcon.service.authentication.kerberos.keytab' not in security_params['startup'] or '*.falcon.service.authentication.kerberos.principal' not in security_params['startup']) \ or '*.falcon.http.authentication.kerberos.keytab' not in security_params['startup'] \ or '*.falcon.http.authentication.kerberos.principal' not in security_params['startup']: self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out( {"securityIssuesFound": "Keytab file or principal are not set property."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.falcon_user, security_params['startup']['*.falcon.service.authentication.kerberos.keytab'], security_params['startup']['*.falcon.service.authentication.kerberos.principal'], status_params.hostname, status_params.tmp_dir) cached_kinit_executor(status_params.kinit_path_local, status_params.falcon_user, security_params['startup']['*.falcon.http.authentication.kerberos.keytab'], security_params['startup']['*.falcon.http.authentication.kerberos.principal'], status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) props_value_check = {} props_empty_check = [ 'general.kerberos.keytab', 'general.kerberos.principal' ] props_read_check = ['general.kerberos.keytab'] accumulo_site_expectations = build_expectations( 'accumulo-site', props_value_check, props_empty_check, props_read_check) accumulo_expectations = {} accumulo_expectations.update(accumulo_site_expectations) security_params = get_params_from_filesystem( status_params.conf_dir, {'accumulo-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties( security_params, accumulo_expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ('accumulo-site' not in security_params or 'general.kerberos.keytab' not in security_params['accumulo-site'] or 'general.kerberos.principal' not in security_params['accumulo-site']): self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out({ "securityIssuesFound": "Keytab file or principal are not set property." }) return cached_kinit_executor( status_params.kinit_path_local, status_params.accumulo_user, security_params['accumulo-site'] ['general.kerberos.keytab'], security_params['accumulo-site'] ['general.kerberos.principal'], status_params.hostname, status_params.tmp_dir, 30) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append( "Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: props_value_check = { "hive.server2.authentication": "KERBEROS", "hive.metastore.sasl.enabled": "true", "hive.security.authorization.enabled": "true", } props_empty_check = ["hive.metastore.kerberos.keytab.file", "hive.metastore.kerberos.principal"] props_read_check = ["hive.metastore.kerberos.keytab.file"] hive_site_props = build_expectations("hive-site", props_value_check, props_empty_check, props_read_check) hive_expectations = {} hive_expectations.update(hive_site_props) security_params = get_params_from_filesystem(status_params.hive_conf_dir, {"hive-site.xml": FILE_TYPE_XML}) result_issues = validate_security_config_properties(security_params, hive_expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ( "hive-site" not in security_params or "hive.metastore.kerberos.keytab.file" not in security_params["hive-site"] or "hive.metastore.kerberos.principal" not in security_params["hive-site"] ): self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out( {"securityIssuesFound": "Keytab file or principal are not set property."} ) return cached_kinit_executor( status_params.kinit_path_local, status_params.hive_user, security_params["hive-site"]["hive.metastore.kerberos.keytab.file"], security_params["hive-site"]["hive.metastore.kerberos.principal"], status_params.hostname, status_params.tmp_dir, ) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append( "Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]) ) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) props_value_check = { "hadoop.security.authentication": "kerberos", "hadoop.security.authorization": "true" } props_empty_check = ["hadoop.security.auth_to_local"] props_read_check = None core_site_expectations = build_expectations('core-site', props_value_check, props_empty_check, props_read_check) hdfs_expectations = {} hdfs_expectations.update(core_site_expectations) security_params = get_params_from_filesystem( status_params.hadoop_conf_dir, {'core-site.xml': FILE_TYPE_XML}) if 'core-site' in security_params and 'hadoop.security.authentication' in security_params['core-site'] and \ security_params['core-site']['hadoop.security.authentication'].lower() == 'kerberos': result_issues = validate_security_config_properties( security_params, hdfs_expectations) if not result_issues: # If all validations passed successfully if status_params.hdfs_user_principal or status_params.hdfs_user_keytab: try: cached_kinit_executor( status_params.kinit_path_local, status_params.hdfs_user, status_params.hdfs_user_keytab, status_params.hdfs_user_principal, status_params.hostname, status_params.tmp_dir) self.put_structured_out( {"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out( {"securityStateErrorInfo": str(e)}) else: self.put_structured_out({ "securityIssuesFound": "hdfs principal and/or keytab file is not specified" }) self.put_structured_out({"securityState": "UNSECURED"}) else: issues = [] for cf in result_issues: issues.append( "Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out( {"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: expectations = {} expectations.update(build_expectations('mapred-site', None, [ 'mapreduce.jobhistory.keytab', 'mapreduce.jobhistory.principal', 'mapreduce.jobhistory.webapp.spnego-keytab-file', 'mapreduce.jobhistory.webapp.spnego-principal' ], None)) security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, {'mapred-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties(security_params, expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ( 'mapred-site' not in security_params or 'mapreduce.jobhistory.keytab' not in security_params['mapred-site'] or 'mapreduce.jobhistory.principal' not in security_params['mapred-site'] or 'mapreduce.jobhistory.webapp.spnego-keytab-file' not in security_params['mapred-site'] or 'mapreduce.jobhistory.webapp.spnego-principal' not in security_params['mapred-site']): self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out( {"securityIssuesFound": "Keytab file or principal not set."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.mapred_user, security_params['mapred-site']['mapreduce.jobhistory.keytab'], security_params['mapred-site']['mapreduce.jobhistory.principal'], status_params.hostname, status_params.tmp_dir) cached_kinit_executor(status_params.kinit_path_local, status_params.mapred_user, security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-keytab-file'], security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-principal'], status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: expectations = {} expectations.update(build_expectations('mapred-site', None, [ 'mapreduce.jobhistory.keytab', 'mapreduce.jobhistory.principal', 'mapreduce.jobhistory.webapp.spnego-keytab-file', 'mapreduce.jobhistory.webapp.spnego-principal' ], None)) security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, {'mapred-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties(security_params, expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ( 'mapred-site' not in security_params or 'mapreduce.jobhistory.keytab' not in security_params['mapred-site'] or 'mapreduce.jobhistory.principal' not in security_params['mapred-site'] or 'mapreduce.jobhistory.webapp.spnego-keytab-file' not in security_params['mapred-site'] or 'mapreduce.jobhistory.webapp.spnego-principal' not in security_params['mapred-site']): self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out( {"securityIssuesFound": "Keytab file or principal not set."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.mapred_user, security_params['mapred-site']['mapreduce.jobhistory.keytab'], security_params['mapred-site']['mapreduce.jobhistory.principal'], status_params.hostname, status_params.tmp_dir) cached_kinit_executor(status_params.kinit_path_local, status_params.mapred_user, security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-keytab-file'], security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-principal'], status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def service_check(self, env): import params env.set_params(params) jar_path = format( "{hadoop_mapred2_jar_location}/{hadoopMapredExamplesJarName}") input_file = format("/user/{smokeuser}/mapredsmokeinput") output_file = format("/user/{smokeuser}/mapredsmokeoutput") test_cmd = format("fs -test -e {output_file}") run_wordcount_job = format( "jar {jar_path} wordcount {input_file} {output_file}") params.HdfsResource( output_file, action="delete_on_execute", type="directory", dfs_type=params.dfs_type, ) params.HdfsResource( input_file, action="create_on_execute", type="file", source="/etc/passwd", dfs_type=params.dfs_type, ) params.HdfsResource(None, action="execute") # initialize the ticket if params.security_enabled: cached_kinit_executor(params.kinit_path_local, params.smokeuser, params.smoke_user_keytab, params.smokeuser_principal, params.hostname, params.tmp_dir) ExecuteHadoop(run_wordcount_job, tries=1, try_sleep=5, user=params.smokeuser, bin_dir=params.execute_path, conf_dir=params.hadoop_conf_dir, logoutput=True) # the ticket may have expired, so re-initialize if needed if params.security_enabled: cached_kinit_executor(params.kinit_path_local, params.smokeuser, params.smoke_user_keytab, params.smokeuser_principal, params.hostname, params.tmp_dir) ExecuteHadoop(test_cmd, user=params.smokeuser, bin_dir=params.execute_path, conf_dir=params.hadoop_conf_dir)
def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: # Expect the following files to be available in status_params.config_dir: # storm_jaas.conf try: props_value_check = None props_empty_check = ['storm_ui_principal_name', 'storm_ui_keytab'] props_read_check = ['storm_ui_keytab'] storm_env_expectations = build_expectations('storm_ui', props_value_check, props_empty_check, props_read_check) storm_expectations = {} storm_expectations.update(storm_env_expectations) security_params = {} security_params['storm_ui'] = {} security_params['storm_ui']['storm_ui_principal_name'] = status_params.storm_ui_principal security_params['storm_ui']['storm_ui_keytab'] = status_params.storm_ui_keytab result_issues = validate_security_config_properties(security_params, storm_expectations) if not result_issues: # If all validations passed successfully # Double check the dict before calling execute if ( 'storm_ui' not in security_params or 'storm_ui_principal_name' not in security_params['storm_ui'] or 'storm_ui_keytab' not in security_params['storm_ui']): self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.storm_user, security_params['storm_ui']['storm_ui_keytab'], security_params['storm_ui']['storm_ui_principal_name'], status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: # Expect the following files to be available in status_params.config_dir: # zookeeper_jaas.conf # zookeeper_client_jaas.conf try: props_value_check = None props_empty_check = ['Server/keyTab', 'Server/principal'] props_read_check = ['Server/keyTab'] zk_env_expectations = build_expectations('zookeeper_jaas', props_value_check, props_empty_check, props_read_check) zk_expectations = {} zk_expectations.update(zk_env_expectations) security_params = get_params_from_filesystem(status_params.config_dir, {'zookeeper_jaas.conf': FILE_TYPE_JAAS_CONF}) result_issues = validate_security_config_properties(security_params, zk_expectations) if not result_issues: # If all validations passed successfully # Double check the dict before calling execute if ( 'zookeeper_jaas' not in security_params or 'Server' not in security_params['zookeeper_jaas'] or 'keyTab' not in security_params['zookeeper_jaas']['Server'] or 'principal' not in security_params['zookeeper_jaas']['Server']): self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.zk_user, security_params['zookeeper_jaas']['Server']['keyTab'], security_params['zookeeper_jaas']['Server']['principal'], status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) props_value_check = {} props_empty_check = ['general.kerberos.keytab', 'general.kerberos.principal'] props_read_check = ['general.kerberos.keytab'] accumulo_site_expectations = build_expectations('accumulo-site', props_value_check, props_empty_check, props_read_check) accumulo_expectations = {} accumulo_expectations.update(accumulo_site_expectations) security_params = get_params_from_filesystem(status_params.conf_dir, {'accumulo-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties(security_params, accumulo_expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ( 'accumulo-site' not in security_params or 'general.kerberos.keytab' not in security_params['accumulo-site'] or 'general.kerberos.principal' not in security_params['accumulo-site']): self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out( {"securityIssuesFound": "Keytab file or principal are not set property."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.accumulo_user, security_params['accumulo-site']['general.kerberos.keytab'], security_params['accumulo-site']['general.kerberos.principal'], status_params.hostname, status_params.tmp_dir, 30) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) props_value_check = {"hadoop.security.authentication": "kerberos", "hadoop.security.authorization": "true"} props_empty_check = ["hadoop.security.auth_to_local"] props_read_check = None core_site_expectations = build_expectations('core-site', props_value_check, props_empty_check, props_read_check) hdfs_expectations = {} hdfs_expectations.update(core_site_expectations) security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, {'core-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties(security_params, hdfs_expectations) if 'core-site' in security_params and 'hadoop.security.authentication' in security_params['core-site'] and \ security_params['core-site']['hadoop.security.authentication'].lower() == 'kerberos': if not result_issues: # If all validations passed successfully if status_params.hdfs_user_principal or status_params.hdfs_user_keytab: try: cached_kinit_executor(status_params.kinit_path_local, status_params.hdfs_user, status_params.hdfs_user_keytab, status_params.hdfs_user_principal, status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: self.put_structured_out( {"securityIssuesFound": "hdfs principal and/or keytab file is not specified"}) self.put_structured_out({"securityState": "UNSECURED"}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params if status_params.security_enabled: if status_params.smoke_user and status_params.smoke_user_keytab: try: cached_kinit_executor(status_params.kinit_path_local, status_params.smoke_user, status_params.smoke_user_keytab, status_params.smoke_user_principal, status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: self.put_structured_out({"securityState": "UNKNOWN"}) self.put_structured_out({"securityStateErrorInfo": "Missing smoke user credentials"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params if status_params.security_enabled: if status_params.smoke_user and status_params.smoke_user_keytab: try: cached_kinit_executor(status_params.kinit_path_local, status_params.smoke_user, status_params.smoke_user_keytab, status_params.smoke_user_principal, status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: self.put_structured_out({"securityState": "UNKNOWN"}) self.put_structured_out({"securityStateErrorInfo": "Missing smoke user credentials"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: expectations ={} expectations.update( build_expectations( 'webhcat-site', { "templeton.kerberos.secret": "secret" }, [ "templeton.kerberos.keytab", "templeton.kerberos.principal" ], [ "templeton.kerberos.keytab" ] ) ) expectations.update( build_expectations( 'hive-site', { "hive.server2.authentication": "KERBEROS", "hive.metastore.sasl.enabled": "true", "hive.security.authorization.enabled": "true" }, None, None ) ) security_params = {} security_params.update(get_params_from_filesystem(status_params.hive_conf_dir, {'hive-site.xml': FILE_TYPE_XML})) security_params.update(get_params_from_filesystem(status_params.webhcat_conf_dir, {'webhcat-site.xml': FILE_TYPE_XML})) result_issues = validate_security_config_properties(security_params, expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if 'webhcat-site' not in security_params \ or 'templeton.kerberos.keytab' not in security_params['webhcat-site'] \ or 'templeton.kerberos.principal' not in security_params['webhcat-site']: self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.webhcat_user, security_params['webhcat-site']['templeton.kerberos.keytab'], security_params['webhcat-site']['templeton.kerberos.principal'], status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def upgrade_schema(self, env): """ Executes the schema upgrade binary. This is its own function because it could be called as a standalone task from the upgrade pack, but is safe to run it for each metastore instance. The schema upgrade on an already upgraded metastore is a NOOP. The metastore schema upgrade requires a database driver library for most databases. During an upgrade, it's possible that the library is not present, so this will also attempt to copy/download the appropriate driver. This function will also ensure that configurations are written out to disk before running since the new configs will most likely not yet exist on an upgrade. Should not be invoked for a DOWNGRADE; Metastore only supports schema upgrades. """ Logger.info("Upgrading Hive Metastore Schema") import status_params import params env.set_params(params) # ensure that configurations are written out before trying to upgrade the schema # since the schematool needs configs and doesn't know how to use the hive conf override self.configure(env) if params.security_enabled: cached_kinit_executor(status_params.kinit_path_local, status_params.hive_user, params.hive_metastore_keytab_path, params.hive_metastore_principal, status_params.hostname, status_params.tmp_dir) # ensure that the JDBC drive is present for the schema tool; if it's not # present, then download it first if params.hive_jdbc_driver in params.hive_jdbc_drivers_list: target_directory = format("{params.install_dir}/lib") # download it if it does not exist if not os.path.exists(params.source_jdbc_file): jdbc_connector(params.hive_jdbc_target, params.hive_previous_jdbc_jar) target_directory_and_filename = os.path.join( target_directory, os.path.basename(params.source_jdbc_file)) # copy the JDBC driver from the older metastore location to the new location only # if it does not already exist if not os.path.exists(target_directory_and_filename): Execute(('cp', params.source_jdbc_file, target_directory), path=["/bin", "/usr/bin/"], sudo=True) File(target_directory_and_filename, mode=0644) # build the schema tool command binary = format("{hive_schematool_ver_bin}/schematool") schematool_hive_server_conf_dir = params.hive_server_conf_dir env_dict = {'HIVE_CONF_DIR': schematool_hive_server_conf_dir} command = format( "{binary} -dbType {hive_metastore_db_type} -upgradeSchema") Execute(command, user=params.hive_user, tries=1, environment=env_dict, logoutput=True)
def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: expectations = { "oozie-site": build_expectations( 'oozie-site', { "oozie.authentication.type": "kerberos", "oozie.service.AuthorizationService.security.enabled": "true", "oozie.service.HadoopAccessorService.kerberos.enabled": "true" }, [ "local.realm", "oozie.authentication.kerberos.principal", "oozie.authentication.kerberos.keytab", "oozie.service.HadoopAccessorService.kerberos.principal", "oozie.service.HadoopAccessorService.keytab.file" ], None) } security_params = get_params_from_filesystem( status_params.conf_dir, {'oozie-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties( security_params, expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ('oozie-site' not in security_params or 'oozie.authentication.kerberos.principal' not in security_params['oozie-site'] or 'oozie.authentication.kerberos.keytab' not in security_params['oozie-site'] or 'oozie.service.HadoopAccessorService.kerberos.principal' not in security_params['oozie-site'] or 'oozie.service.HadoopAccessorService.keytab.file' not in security_params['oozie-site']): self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out({ "securityIssuesFound": "Keytab file or principal are not set property." }) return cached_kinit_executor( status_params.kinit_path_local, status_params.oozie_user, security_params['oozie-site'] ['oozie.authentication.kerberos.keytab'], security_params['oozie-site'] ['oozie.authentication.kerberos.principal'], status_params.hostname, status_params.tmp_dir) cached_kinit_executor( status_params.kinit_path_local, status_params.oozie_user, security_params['oozie-site'] ['oozie.service.HadoopAccessorService.keytab.file'], security_params['oozie-site'] ['oozie.service.HadoopAccessorService.kerberos.principal'], status_params.hostname, status_params.tmp_dir) self.put_structured_out( {"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append( "Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out( {"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})
def upgrade_schema(self, env): """ Executes the schema upgrade binary. This is its own function because it could be called as a standalone task from the upgrade pack, but is safe to run it for each metastore instance. The schema upgrade on an already upgraded metastore is a NOOP. The metastore schema upgrade requires a database driver library for most databases. During an upgrade, it's possible that the library is not present, so this will also attempt to copy/download the appropriate driver. This function will also ensure that configurations are written out to disk before running since the new configs will most likely not yet exist on an upgrade. Should not be invoked for a DOWNGRADE; Metastore only supports schema upgrades. """ Logger.info("Upgrading Hive Metastore Schema") import status_params import params env.set_params(params) # ensure that configurations are written out before trying to upgrade the schema # since the schematool needs configs and doesn't know how to use the hive conf override self.configure(env) if params.security_enabled: cached_kinit_executor(status_params.kinit_path_local, status_params.hive_user, params.hive_metastore_keytab_path, params.hive_metastore_principal, status_params.hostname, status_params.tmp_dir) # ensure that the JDBC drive is present for the schema tool; if it's not # present, then download it first if params.hive_jdbc_driver in params.hive_jdbc_drivers_list: target_directory = format("{stack_root}/{version}/hive/lib") # download it if it does not exist if not os.path.exists(params.source_jdbc_file): jdbc_connector(params.hive_jdbc_target, params.hive_previous_jdbc_jar) target_directory_and_filename = os.path.join( target_directory, os.path.basename(params.source_jdbc_file)) if params.sqla_db_used: target_native_libs_directory = format( "{target_directory}/native/lib64") Execute( format( "yes | {sudo} cp {jars_in_hive_lib} {target_directory}" )) Directory(target_native_libs_directory, create_parents=True) Execute( format( "yes | {sudo} cp {libs_in_hive_lib} {target_native_libs_directory}" )) Execute( format( "{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*" )) else: # copy the JDBC driver from the older metastore location to the new location only # if it does not already exist if not os.path.exists(target_directory_and_filename): Execute(('cp', params.source_jdbc_file, target_directory), path=["/bin", "/usr/bin/"], sudo=True) File(target_directory_and_filename, mode=0644) # build the schema tool command binary = format("{hive_schematool_ver_bin}/schematool") # the conf.server directory changed locations between stack versions # since the configurations have not been written out yet during an upgrade # we need to choose the original legacy location schematool_hive_server_conf_dir = params.hive_server_conf_dir if params.current_version is not None: current_version = format_stack_version(params.current_version) if not (check_stack_feature(StackFeature.CONFIG_VERSIONING, current_version)): schematool_hive_server_conf_dir = LEGACY_HIVE_SERVER_CONF env_dict = {'HIVE_CONF_DIR': schematool_hive_server_conf_dir} command = format( "{binary} -dbType {hive_metastore_db_type} -upgradeSchema") Execute(command, user=params.hive_user, tries=1, environment=env_dict, logoutput=True)
def security_status(self, env): import status_params env.set_params(status_params) if status_params.security_enabled: expectations = { "oozie-site": build_expectations('oozie-site', { "oozie.authentication.type": "kerberos", "oozie.service.AuthorizationService.security.enabled": "true", "oozie.service.HadoopAccessorService.kerberos.enabled": "true" }, [ "local.realm", "oozie.authentication.kerberos.principal", "oozie.authentication.kerberos.keytab", "oozie.service.HadoopAccessorService.kerberos.principal", "oozie.service.HadoopAccessorService.keytab.file" ], None) } security_params = get_params_from_filesystem(status_params.conf_dir, {'oozie-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties(security_params, expectations) if not result_issues: # If all validations passed successfully try: # Double check the dict before calling execute if ('oozie-site' not in security_params or 'oozie.authentication.kerberos.principal' not in security_params['oozie-site'] or 'oozie.authentication.kerberos.keytab' not in security_params['oozie-site'] or 'oozie.service.HadoopAccessorService.kerberos.principal' not in security_params['oozie-site'] or 'oozie.service.HadoopAccessorService.keytab.file' not in security_params['oozie-site']): self.put_structured_out({"securityState": "UNSECURED"}) self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."}) return cached_kinit_executor(status_params.kinit_path_local, status_params.oozie_user, security_params['oozie-site']['oozie.authentication.kerberos.keytab'], security_params['oozie-site']['oozie.authentication.kerberos.principal'], status_params.hostname, status_params.tmp_dir) cached_kinit_executor(status_params.kinit_path_local, status_params.oozie_user, security_params['oozie-site']['oozie.service.HadoopAccessorService.keytab.file'], security_params['oozie-site']['oozie.service.HadoopAccessorService.kerberos.principal'], status_params.hostname, status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: issues = [] for cf in result_issues: issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"})