def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
expectations = {}
expectations.update(build_expectations(
'krb5JAASLogin',
None,
['keytab', 'principal'],
None
))
expectations.update(build_expectations(
'gateway-site',
{
"gateway.hadoop.kerberos.secured" : "true"
},
None,
None
))
security_params = {
"krb5JAASLogin":
{
'keytab': status_params.knox_keytab_path,
'principal': status_params.knox_principal_name
}
}
security_params.update(get_params_from_filesystem(status_params.knox_conf_dir,
{"gateway-site.xml" : FILE_TYPE_XML}))
result_issues = validate_security_config_properties(security_params, expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if ( 'krb5JAASLogin' not in security_params
or 'keytab' not in security_params['krb5JAASLogin']
or 'principal' not in security_params['krb5JAASLogin']):
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out({"securityIssuesFound": "Keytab file and principal are not set."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.knox_user,
security_params['krb5JAASLogin']['keytab'],
security_params['krb5JAASLogin']['principal'],
status_params.hostname,
status_params.temp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
props_value_check = {"yarn.timeline-service.enabled": "true",
"yarn.timeline-service.http-authentication.type": "kerberos",
"yarn.acl.enable": "true"}
props_empty_check = ["yarn.timeline-service.principal",
"yarn.timeline-service.keytab",
"yarn.timeline-service.http-authentication.kerberos.principal",
"yarn.timeline-service.http-authentication.kerberos.keytab"]
props_read_check = ["yarn.timeline-service.keytab",
"yarn.timeline-service.http-authentication.kerberos.keytab"]
yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check,
props_read_check)
yarn_expectations ={}
yarn_expectations.update(yarn_site_props)
security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
{'yarn-site.xml': FILE_TYPE_XML})
result_issues = validate_security_config_properties(security_params, yarn_expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if ( 'yarn-site' not in security_params
or 'yarn.timeline-service.keytab' not in security_params['yarn-site']
or 'yarn.timeline-service.principal' not in security_params['yarn-site']) \
or 'yarn.timeline-service.http-authentication.kerberos.keytab' not in security_params['yarn-site'] \
or 'yarn.timeline-service.http-authentication.kerberos.principal' not in security_params['yarn-site']:
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out(
{"securityIssuesFound": "Keytab file or principal are not set property."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.yarn_user,
security_params['yarn-site']['yarn.timeline-service.keytab'],
security_params['yarn-site']['yarn.timeline-service.principal'],
status_params.hostname,
status_params.tmp_dir)
cached_kinit_executor(status_params.kinit_path_local,
status_params.yarn_user,
security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.keytab'],
security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.principal'],
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
expectations = {}
expectations.update(build_expectations(
'krb5JAASLogin',
None,
['keytab', 'principal'],
None
))
expectations.update(build_expectations(
'gateway-site',
{
"gateway.hadoop.kerberos.secured" : "true"
},
None,
None
))
security_params = {
"krb5JAASLogin":
{
'keytab': status_params.knox_keytab_path,
'principal': status_params.knox_principal_name
}
}
security_params.update(get_params_from_filesystem(status_params.knox_conf_dir,
{"gateway-site.xml" : FILE_TYPE_XML}))
result_issues = validate_security_config_properties(security_params, expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if ( 'krb5JAASLogin' not in security_params
or 'keytab' not in security_params['krb5JAASLogin']
or 'principal' not in security_params['krb5JAASLogin']):
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out({"securityIssuesFound": "Keytab file and principal are not set."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.knox_user,
security_params['krb5JAASLogin']['keytab'],
security_params['krb5JAASLogin']['principal'],
status_params.hostname,
status_params.temp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
props_value_check = {"hbase.security.authentication": "kerberos",
"hbase.security.authorization": "true"}
props_empty_check = ["hbase.zookeeper.property.authProvider.1",
"hbase.master.keytab.file",
"hbase.master.kerberos.principal",
"hbase.regionserver.keytab.file",
"hbase.regionserver.kerberos.principal"
]
props_read_check = ['hbase.master.keytab.file', 'hbase.regionserver.keytab.file']
ams_hbase_site_expectations = build_expectations('hbase-site', props_value_check,
props_empty_check,
props_read_check)
expectations = {}
expectations.update(ams_hbase_site_expectations)
security_params = get_params_from_filesystem(status_params.ams_hbase_conf_dir,
{'hbase-site.xml': FILE_TYPE_XML})
is_hbase_distributed = security_params['hbase-site']['hbase.cluster.distributed']
# for embedded mode, when HBase is backed by file, security state is SECURED_KERBEROS by definition when cluster is secured
if status_params.security_enabled and not is_hbase_distributed:
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
return
result_issues = validate_security_config_properties(security_params, expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if ('hbase-site' not in security_params or
'hbase.master.keytab.file' not in security_params['hbase-site'] or
'hbase.master.kerberos.principal' not in security_params['hbase-site']):
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out(
{"securityIssuesFound": "Keytab file or principal are not set property."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.hbase_user,
security_params['hbase-site']['hbase.master.keytab.file'],
security_params['hbase-site']['hbase.master.kerberos.principal'],
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (
cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
props_value_check = {"yarn.timeline-service.enabled": "true",
"yarn.timeline-service.http-authentication.type": "kerberos",
"yarn.acl.enable": "true"}
props_empty_check = ["yarn.timeline-service.principal",
"yarn.timeline-service.keytab",
"yarn.timeline-service.http-authentication.kerberos.principal",
"yarn.timeline-service.http-authentication.kerberos.keytab"]
props_read_check = ["yarn.timeline-service.keytab",
"yarn.timeline-service.http-authentication.kerberos.keytab"]
yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check,
props_read_check)
yarn_expectations ={}
yarn_expectations.update(yarn_site_props)
security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
{'yarn-site.xml': FILE_TYPE_XML})
result_issues = validate_security_config_properties(security_params, yarn_expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if ( 'yarn-site' not in security_params
or 'yarn.timeline-service.keytab' not in security_params['yarn-site']
or 'yarn.timeline-service.principal' not in security_params['yarn-site']) \
or 'yarn.timeline-service.http-authentication.kerberos.keytab' not in security_params['yarn-site'] \
or 'yarn.timeline-service.http-authentication.kerberos.principal' not in security_params['yarn-site']:
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out(
{"securityIssuesFound": "Keytab file or principal are not set property."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.yarn_user,
security_params['yarn-site']['yarn.timeline-service.keytab'],
security_params['yarn-site']['yarn.timeline-service.principal'],
status_params.hostname,
status_params.tmp_dir)
cached_kinit_executor(status_params.kinit_path_local,
status_params.yarn_user,
security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.keytab'],
security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.principal'],
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
props_value_check = {"hbase.security.authentication": "kerberos",
"hbase.security.authorization": "true"}
props_empty_check = ["hbase.zookeeper.property.authProvider.1",
"hbase.master.keytab.file",
"hbase.master.kerberos.principal",
"hbase.regionserver.keytab.file",
"hbase.regionserver.kerberos.principal"
]
props_read_check = ['hbase.master.keytab.file', 'hbase.regionserver.keytab.file']
ams_hbase_site_expectations = build_expectations('hbase-site', props_value_check,
props_empty_check,
props_read_check)
expectations = {}
expectations.update(ams_hbase_site_expectations)
security_params = get_params_from_filesystem(status_params.ams_hbase_conf_dir,
{'hbase-site.xml': FILE_TYPE_XML})
is_hbase_distributed = security_params['hbase-site']['hbase.cluster.distributed']
# for embedded mode, when HBase is backed by file, security state is SECURED_KERBEROS by definition when cluster is secured
if status_params.security_enabled and not is_hbase_distributed:
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
return
result_issues = validate_security_config_properties(security_params, expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if ('hbase-site' not in security_params or
'hbase.master.keytab.file' not in security_params['hbase-site'] or
'hbase.master.kerberos.principal' not in security_params['hbase-site']):
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out(
{"securityIssuesFound": "Keytab file or principal are not set property."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.hbase_user,
security_params['hbase-site']['hbase.master.keytab.file'],
security_params['hbase-site']['hbase.master.kerberos.principal'],
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (
cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
props_value_check = {"hive.server2.authentication": "KERBEROS",
"hive.metastore.sasl.enabled": "true",
"hive.security.authorization.enabled": "true"}
props_empty_check = ["hive.server2.authentication.kerberos.keytab",
"hive.server2.authentication.kerberos.principal",
"hive.server2.authentication.spnego.principal",
"hive.server2.authentication.spnego.keytab"]
props_read_check = ["hive.server2.authentication.kerberos.keytab",
"hive.server2.authentication.spnego.keytab"]
hive_site_props = build_expectations('hive-site', props_value_check, props_empty_check,
props_read_check)
hive_expectations ={}
hive_expectations.update(hive_site_props)
security_params = get_params_from_filesystem(status_params.hive_server_interactive_conf_dir,
{'hive-site.xml': FILE_TYPE_XML})
result_issues = validate_security_config_properties(security_params, hive_expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if 'hive-site' not in security_params \
or 'hive.server2.authentication.kerberos.keytab' not in security_params['hive-site'] \
or 'hive.server2.authentication.kerberos.principal' not in security_params['hive-site'] \
or 'hive.server2.authentication.spnego.keytab' not in security_params['hive-site'] \
or 'hive.server2.authentication.spnego.principal' not in security_params['hive-site']:
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.hive_user,
security_params['hive-site']['hive.server2.authentication.kerberos.keytab'],
security_params['hive-site']['hive.server2.authentication.kerberos.principal'],
status_params.hostname,
status_params.tmp_dir)
cached_kinit_executor(status_params.kinit_path_local,
status_params.hive_user,
security_params['hive-site']['hive.server2.authentication.spnego.keytab'],
security_params['hive-site']['hive.server2.authentication.spnego.principal'],
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
# Expect the following files to be available in status_params.config_dir:
# storm_jaas.conf
try:
props_value_check = None
props_empty_check = [
'StormServer/keyTab', 'StormServer/principal'
]
props_read_check = ['StormServer/keyTab']
storm_env_expectations = build_expectations(
'storm_jaas', props_value_check, props_empty_check,
props_read_check)
storm_expectations = {}
storm_expectations.update(storm_env_expectations)
security_params = get_params_from_filesystem(
status_params.conf_dir,
{'storm_jaas.conf': FILE_TYPE_JAAS_CONF})
result_issues = validate_security_config_properties(
security_params, storm_expectations)
if not result_issues: # If all validations passed successfully
# Double check the dict before calling execute
if ('storm_jaas' not in security_params or 'StormServer'
not in security_params['storm_jaas'] or 'keyTab'
not in security_params['storm_jaas']['StormServer']
or 'principal' not in security_params['storm_jaas']
['StormServer']):
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({
"securityIssuesFound":
"Keytab file or principal are not set property."
})
return
cached_kinit_executor(
status_params.kinit_path_local,
status_params.storm_user,
security_params['storm_jaas']['StormServer']['keyTab'],
security_params['storm_jaas']['StormServer']
['principal'], status_params.hostname,
status_params.tmp_dir)
self.put_structured_out(
{"securityState": "SECURED_KERBEROS"})
else:
issues = []
for cf in result_issues:
issues.append(
"Configuration file %s did not pass the validation. Reason: %s"
% (cf, result_issues[cf]))
self.put_structured_out(
{"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
props_value_check = {"hadoop.security.authentication": "kerberos",
"hadoop.security.authorization": "true"}
props_empty_check = ["hadoop.security.auth_to_local"]
props_read_check = None
core_site_expectations = build_expectations('core-site', props_value_check, props_empty_check,
props_read_check)
props_value_check = None
props_empty_check = ['dfs.secondary.namenode.kerberos.internal.spnego.principal',
'dfs.secondary.namenode.keytab.file',
'dfs.secondary.namenode.kerberos.principal']
props_read_check = ['dfs.secondary.namenode.keytab.file']
hdfs_site_expectations = build_expectations('hdfs-site', props_value_check, props_empty_check,
props_read_check)
hdfs_expectations = {}
hdfs_expectations.update(core_site_expectations)
hdfs_expectations.update(hdfs_site_expectations)
security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
{'core-site.xml': FILE_TYPE_XML,
'hdfs-site.xml': FILE_TYPE_XML})
if 'core-site' in security_params and 'hadoop.security.authentication' in security_params['core-site'] and \
security_params['core-site']['hadoop.security.authentication'].lower() == 'kerberos':
result_issues = validate_security_config_properties(security_params, hdfs_expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if ('hdfs-site' not in security_params or
'dfs.secondary.namenode.keytab.file' not in security_params['hdfs-site'] or
'dfs.secondary.namenode.kerberos.principal' not in security_params['hdfs-site']):
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out(
{"securityIssuesFound": "Keytab file or principal are not set property."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.hdfs_user,
security_params['hdfs-site']['dfs.secondary.namenode.keytab.file'],
security_params['hdfs-site'][
'dfs.secondary.namenode.kerberos.principal'],
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
props_value_check = {"*.falcon.authentication.type": "kerberos",
"*.falcon.http.authentication.type": "kerberos"}
props_empty_check = ["*.falcon.service.authentication.kerberos.principal",
"*.falcon.service.authentication.kerberos.keytab",
"*.falcon.http.authentication.kerberos.principal",
"*.falcon.http.authentication.kerberos.keytab"]
props_read_check = ["*.falcon.service.authentication.kerberos.keytab",
"*.falcon.http.authentication.kerberos.keytab"]
falcon_startup_props = build_expectations('startup', props_value_check, props_empty_check,
props_read_check)
falcon_expectations ={}
falcon_expectations.update(falcon_startup_props)
security_params = get_params_from_filesystem('/etc/falcon/conf',
{'startup.properties': FILE_TYPE_PROPERTIES})
result_issues = validate_security_config_properties(security_params, falcon_expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if ( 'startup' not in security_params
or '*.falcon.service.authentication.kerberos.keytab' not in security_params['startup']
or '*.falcon.service.authentication.kerberos.principal' not in security_params['startup']) \
or '*.falcon.http.authentication.kerberos.keytab' not in security_params['startup'] \
or '*.falcon.http.authentication.kerberos.principal' not in security_params['startup']:
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out(
{"securityIssuesFound": "Keytab file or principal are not set property."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.falcon_user,
security_params['startup']['*.falcon.service.authentication.kerberos.keytab'],
security_params['startup']['*.falcon.service.authentication.kerberos.principal'],
status_params.hostname,
status_params.tmp_dir)
cached_kinit_executor(status_params.kinit_path_local,
status_params.falcon_user,
security_params['startup']['*.falcon.http.authentication.kerberos.keytab'],
security_params['startup']['*.falcon.http.authentication.kerberos.principal'],
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
props_value_check = {"*.falcon.authentication.type": "kerberos",
"*.falcon.http.authentication.type": "kerberos"}
props_empty_check = ["*.falcon.service.authentication.kerberos.principal",
"*.falcon.service.authentication.kerberos.keytab",
"*.falcon.http.authentication.kerberos.principal",
"*.falcon.http.authentication.kerberos.keytab"]
props_read_check = ["*.falcon.service.authentication.kerberos.keytab",
"*.falcon.http.authentication.kerberos.keytab"]
falcon_startup_props = build_expectations('startup', props_value_check, props_empty_check,
props_read_check)
falcon_expectations ={}
falcon_expectations.update(falcon_startup_props)
security_params = get_params_from_filesystem('/etc/falcon/conf',
{'startup.properties': FILE_TYPE_PROPERTIES})
result_issues = validate_security_config_properties(security_params, falcon_expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if ( 'startup' not in security_params
or '*.falcon.service.authentication.kerberos.keytab' not in security_params['startup']
or '*.falcon.service.authentication.kerberos.principal' not in security_params['startup']) \
or '*.falcon.http.authentication.kerberos.keytab' not in security_params['startup'] \
or '*.falcon.http.authentication.kerberos.principal' not in security_params['startup']:
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out(
{"securityIssuesFound": "Keytab file or principal are not set property."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.falcon_user,
security_params['startup']['*.falcon.service.authentication.kerberos.keytab'],
security_params['startup']['*.falcon.service.authentication.kerberos.principal'],
status_params.hostname,
status_params.tmp_dir)
cached_kinit_executor(status_params.kinit_path_local,
status_params.falcon_user,
security_params['startup']['*.falcon.http.authentication.kerberos.keytab'],
security_params['startup']['*.falcon.http.authentication.kerberos.principal'],
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
props_value_check = {}
props_empty_check = [
'general.kerberos.keytab', 'general.kerberos.principal'
]
props_read_check = ['general.kerberos.keytab']
accumulo_site_expectations = build_expectations(
'accumulo-site', props_value_check, props_empty_check,
props_read_check)
accumulo_expectations = {}
accumulo_expectations.update(accumulo_site_expectations)
security_params = get_params_from_filesystem(
status_params.conf_dir, {'accumulo-site.xml': FILE_TYPE_XML})
result_issues = validate_security_config_properties(
security_params, accumulo_expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if ('accumulo-site' not in security_params
or 'general.kerberos.keytab'
not in security_params['accumulo-site']
or 'general.kerberos.principal'
not in security_params['accumulo-site']):
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out({
"securityIssuesFound":
"Keytab file or principal are not set property."
})
return
cached_kinit_executor(
status_params.kinit_path_local,
status_params.accumulo_user,
security_params['accumulo-site']
['general.kerberos.keytab'],
security_params['accumulo-site']
['general.kerberos.principal'], status_params.hostname,
status_params.tmp_dir, 30)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append(
"Configuration file %s did not pass the validation. Reason: %s"
% (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
props_value_check = {
"hive.server2.authentication": "KERBEROS",
"hive.metastore.sasl.enabled": "true",
"hive.security.authorization.enabled": "true",
}
props_empty_check = ["hive.metastore.kerberos.keytab.file", "hive.metastore.kerberos.principal"]
props_read_check = ["hive.metastore.kerberos.keytab.file"]
hive_site_props = build_expectations("hive-site", props_value_check, props_empty_check, props_read_check)
hive_expectations = {}
hive_expectations.update(hive_site_props)
security_params = get_params_from_filesystem(status_params.hive_conf_dir, {"hive-site.xml": FILE_TYPE_XML})
result_issues = validate_security_config_properties(security_params, hive_expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if (
"hive-site" not in security_params
or "hive.metastore.kerberos.keytab.file" not in security_params["hive-site"]
or "hive.metastore.kerberos.principal" not in security_params["hive-site"]
):
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out(
{"securityIssuesFound": "Keytab file or principal are not set property."}
)
return
cached_kinit_executor(
status_params.kinit_path_local,
status_params.hive_user,
security_params["hive-site"]["hive.metastore.kerberos.keytab.file"],
security_params["hive-site"]["hive.metastore.kerberos.principal"],
status_params.hostname,
status_params.tmp_dir,
)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append(
"Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])
)
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
props_value_check = {
"hadoop.security.authentication": "kerberos",
"hadoop.security.authorization": "true"
}
props_empty_check = ["hadoop.security.auth_to_local"]
props_read_check = None
core_site_expectations = build_expectations('core-site',
props_value_check,
props_empty_check,
props_read_check)
hdfs_expectations = {}
hdfs_expectations.update(core_site_expectations)
security_params = get_params_from_filesystem(
status_params.hadoop_conf_dir, {'core-site.xml': FILE_TYPE_XML})
if 'core-site' in security_params and 'hadoop.security.authentication' in security_params['core-site'] and \
security_params['core-site']['hadoop.security.authentication'].lower() == 'kerberos':
result_issues = validate_security_config_properties(
security_params, hdfs_expectations)
if not result_issues: # If all validations passed successfully
if status_params.hdfs_user_principal or status_params.hdfs_user_keytab:
try:
cached_kinit_executor(
status_params.kinit_path_local,
status_params.hdfs_user,
status_params.hdfs_user_keytab,
status_params.hdfs_user_principal,
status_params.hostname, status_params.tmp_dir)
self.put_structured_out(
{"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out(
{"securityStateErrorInfo": str(e)})
else:
self.put_structured_out({
"securityIssuesFound":
"hdfs principal and/or keytab file is not specified"
})
self.put_structured_out({"securityState": "UNSECURED"})
else:
issues = []
for cf in result_issues:
issues.append(
"Configuration file %s did not pass the validation. Reason: %s"
% (cf, result_issues[cf]))
self.put_structured_out(
{"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
expectations = {}
expectations.update(build_expectations('mapred-site',
None,
[
'mapreduce.jobhistory.keytab',
'mapreduce.jobhistory.principal',
'mapreduce.jobhistory.webapp.spnego-keytab-file',
'mapreduce.jobhistory.webapp.spnego-principal'
],
None))
security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
{'mapred-site.xml': FILE_TYPE_XML})
result_issues = validate_security_config_properties(security_params, expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if ( 'mapred-site' not in security_params or
'mapreduce.jobhistory.keytab' not in security_params['mapred-site'] or
'mapreduce.jobhistory.principal' not in security_params['mapred-site'] or
'mapreduce.jobhistory.webapp.spnego-keytab-file' not in security_params['mapred-site'] or
'mapreduce.jobhistory.webapp.spnego-principal' not in security_params['mapred-site']):
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out(
{"securityIssuesFound": "Keytab file or principal not set."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.mapred_user,
security_params['mapred-site']['mapreduce.jobhistory.keytab'],
security_params['mapred-site']['mapreduce.jobhistory.principal'],
status_params.hostname,
status_params.tmp_dir)
cached_kinit_executor(status_params.kinit_path_local,
status_params.mapred_user,
security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-keytab-file'],
security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-principal'],
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
expectations = {}
expectations.update(build_expectations('mapred-site',
None,
[
'mapreduce.jobhistory.keytab',
'mapreduce.jobhistory.principal',
'mapreduce.jobhistory.webapp.spnego-keytab-file',
'mapreduce.jobhistory.webapp.spnego-principal'
],
None))
security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
{'mapred-site.xml': FILE_TYPE_XML})
result_issues = validate_security_config_properties(security_params, expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if ( 'mapred-site' not in security_params or
'mapreduce.jobhistory.keytab' not in security_params['mapred-site'] or
'mapreduce.jobhistory.principal' not in security_params['mapred-site'] or
'mapreduce.jobhistory.webapp.spnego-keytab-file' not in security_params['mapred-site'] or
'mapreduce.jobhistory.webapp.spnego-principal' not in security_params['mapred-site']):
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out(
{"securityIssuesFound": "Keytab file or principal not set."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.mapred_user,
security_params['mapred-site']['mapreduce.jobhistory.keytab'],
security_params['mapred-site']['mapreduce.jobhistory.principal'],
status_params.hostname,
status_params.tmp_dir)
cached_kinit_executor(status_params.kinit_path_local,
status_params.mapred_user,
security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-keytab-file'],
security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-principal'],
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def service_check(self, env):
import params
env.set_params(params)
jar_path = format(
"{hadoop_mapred2_jar_location}/{hadoopMapredExamplesJarName}")
input_file = format("/user/{smokeuser}/mapredsmokeinput")
output_file = format("/user/{smokeuser}/mapredsmokeoutput")
test_cmd = format("fs -test -e {output_file}")
run_wordcount_job = format(
"jar {jar_path} wordcount {input_file} {output_file}")
params.HdfsResource(
output_file,
action="delete_on_execute",
type="directory",
dfs_type=params.dfs_type,
)
params.HdfsResource(
input_file,
action="create_on_execute",
type="file",
source="/etc/passwd",
dfs_type=params.dfs_type,
)
params.HdfsResource(None, action="execute")
# initialize the ticket
if params.security_enabled:
cached_kinit_executor(params.kinit_path_local, params.smokeuser,
params.smoke_user_keytab,
params.smokeuser_principal, params.hostname,
params.tmp_dir)
ExecuteHadoop(run_wordcount_job,
tries=1,
try_sleep=5,
user=params.smokeuser,
bin_dir=params.execute_path,
conf_dir=params.hadoop_conf_dir,
logoutput=True)
# the ticket may have expired, so re-initialize if needed
if params.security_enabled:
cached_kinit_executor(params.kinit_path_local, params.smokeuser,
params.smoke_user_keytab,
params.smokeuser_principal, params.hostname,
params.tmp_dir)
ExecuteHadoop(test_cmd,
user=params.smokeuser,
bin_dir=params.execute_path,
conf_dir=params.hadoop_conf_dir)
def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
# Expect the following files to be available in status_params.config_dir:
# storm_jaas.conf
try:
props_value_check = None
props_empty_check = ['storm_ui_principal_name', 'storm_ui_keytab']
props_read_check = ['storm_ui_keytab']
storm_env_expectations = build_expectations('storm_ui', props_value_check, props_empty_check,
props_read_check)
storm_expectations = {}
storm_expectations.update(storm_env_expectations)
security_params = {}
security_params['storm_ui'] = {}
security_params['storm_ui']['storm_ui_principal_name'] = status_params.storm_ui_principal
security_params['storm_ui']['storm_ui_keytab'] = status_params.storm_ui_keytab
result_issues = validate_security_config_properties(security_params, storm_expectations)
if not result_issues: # If all validations passed successfully
# Double check the dict before calling execute
if ( 'storm_ui' not in security_params
or 'storm_ui_principal_name' not in security_params['storm_ui']
or 'storm_ui_keytab' not in security_params['storm_ui']):
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.storm_user,
security_params['storm_ui']['storm_ui_keytab'],
security_params['storm_ui']['storm_ui_principal_name'],
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
# Expect the following files to be available in status_params.config_dir:
# zookeeper_jaas.conf
# zookeeper_client_jaas.conf
try:
props_value_check = None
props_empty_check = ['Server/keyTab', 'Server/principal']
props_read_check = ['Server/keyTab']
zk_env_expectations = build_expectations('zookeeper_jaas', props_value_check, props_empty_check,
props_read_check)
zk_expectations = {}
zk_expectations.update(zk_env_expectations)
security_params = get_params_from_filesystem(status_params.config_dir,
{'zookeeper_jaas.conf': FILE_TYPE_JAAS_CONF})
result_issues = validate_security_config_properties(security_params, zk_expectations)
if not result_issues: # If all validations passed successfully
# Double check the dict before calling execute
if ( 'zookeeper_jaas' not in security_params
or 'Server' not in security_params['zookeeper_jaas']
or 'keyTab' not in security_params['zookeeper_jaas']['Server']
or 'principal' not in security_params['zookeeper_jaas']['Server']):
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.zk_user,
security_params['zookeeper_jaas']['Server']['keyTab'],
security_params['zookeeper_jaas']['Server']['principal'],
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
props_value_check = {}
props_empty_check = ['general.kerberos.keytab',
'general.kerberos.principal']
props_read_check = ['general.kerberos.keytab']
accumulo_site_expectations = build_expectations('accumulo-site',
props_value_check, props_empty_check, props_read_check)
accumulo_expectations = {}
accumulo_expectations.update(accumulo_site_expectations)
security_params = get_params_from_filesystem(status_params.conf_dir,
{'accumulo-site.xml': FILE_TYPE_XML})
result_issues = validate_security_config_properties(security_params, accumulo_expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if ( 'accumulo-site' not in security_params
or 'general.kerberos.keytab' not in security_params['accumulo-site']
or 'general.kerberos.principal' not in security_params['accumulo-site']):
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out(
{"securityIssuesFound": "Keytab file or principal are not set property."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.accumulo_user,
security_params['accumulo-site']['general.kerberos.keytab'],
security_params['accumulo-site']['general.kerberos.principal'],
status_params.hostname,
status_params.tmp_dir,
30)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
props_value_check = {"hadoop.security.authentication": "kerberos",
"hadoop.security.authorization": "true"}
props_empty_check = ["hadoop.security.auth_to_local"]
props_read_check = None
core_site_expectations = build_expectations('core-site', props_value_check, props_empty_check,
props_read_check)
hdfs_expectations = {}
hdfs_expectations.update(core_site_expectations)
security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
{'core-site.xml': FILE_TYPE_XML})
result_issues = validate_security_config_properties(security_params, hdfs_expectations)
if 'core-site' in security_params and 'hadoop.security.authentication' in security_params['core-site'] and \
security_params['core-site']['hadoop.security.authentication'].lower() == 'kerberos':
if not result_issues: # If all validations passed successfully
if status_params.hdfs_user_principal or status_params.hdfs_user_keytab:
try:
cached_kinit_executor(status_params.kinit_path_local,
status_params.hdfs_user,
status_params.hdfs_user_keytab,
status_params.hdfs_user_principal,
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
self.put_structured_out(
{"securityIssuesFound": "hdfs principal and/or keytab file is not specified"})
self.put_structured_out({"securityState": "UNSECURED"})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
if status_params.security_enabled:
if status_params.smoke_user and status_params.smoke_user_keytab:
try:
cached_kinit_executor(status_params.kinit_path_local,
status_params.smoke_user,
status_params.smoke_user_keytab,
status_params.smoke_user_principal,
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
self.put_structured_out({"securityState": "UNKNOWN"})
self.put_structured_out({"securityStateErrorInfo": "Missing smoke user credentials"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
if status_params.security_enabled:
if status_params.smoke_user and status_params.smoke_user_keytab:
try:
cached_kinit_executor(status_params.kinit_path_local,
status_params.smoke_user,
status_params.smoke_user_keytab,
status_params.smoke_user_principal,
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
self.put_structured_out({"securityState": "UNKNOWN"})
self.put_structured_out({"securityStateErrorInfo": "Missing smoke user credentials"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
expectations ={}
expectations.update(
build_expectations(
'webhcat-site',
{
"templeton.kerberos.secret": "secret"
},
[
"templeton.kerberos.keytab",
"templeton.kerberos.principal"
],
[
"templeton.kerberos.keytab"
]
)
)
expectations.update(
build_expectations(
'hive-site',
{
"hive.server2.authentication": "KERBEROS",
"hive.metastore.sasl.enabled": "true",
"hive.security.authorization.enabled": "true"
},
None,
None
)
)
security_params = {}
security_params.update(get_params_from_filesystem(status_params.hive_conf_dir,
{'hive-site.xml': FILE_TYPE_XML}))
security_params.update(get_params_from_filesystem(status_params.webhcat_conf_dir,
{'webhcat-site.xml': FILE_TYPE_XML}))
result_issues = validate_security_config_properties(security_params, expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if 'webhcat-site' not in security_params \
or 'templeton.kerberos.keytab' not in security_params['webhcat-site'] \
or 'templeton.kerberos.principal' not in security_params['webhcat-site']:
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.webhcat_user,
security_params['webhcat-site']['templeton.kerberos.keytab'],
security_params['webhcat-site']['templeton.kerberos.principal'],
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def upgrade_schema(self, env):
"""
Executes the schema upgrade binary. This is its own function because it could
be called as a standalone task from the upgrade pack, but is safe to run it for each
metastore instance. The schema upgrade on an already upgraded metastore is a NOOP.
The metastore schema upgrade requires a database driver library for most
databases. During an upgrade, it's possible that the library is not present,
so this will also attempt to copy/download the appropriate driver.
This function will also ensure that configurations are written out to disk before running
since the new configs will most likely not yet exist on an upgrade.
Should not be invoked for a DOWNGRADE; Metastore only supports schema upgrades.
"""
Logger.info("Upgrading Hive Metastore Schema")
import status_params
import params
env.set_params(params)
# ensure that configurations are written out before trying to upgrade the schema
# since the schematool needs configs and doesn't know how to use the hive conf override
self.configure(env)
if params.security_enabled:
cached_kinit_executor(status_params.kinit_path_local,
status_params.hive_user,
params.hive_metastore_keytab_path,
params.hive_metastore_principal,
status_params.hostname,
status_params.tmp_dir)
# ensure that the JDBC drive is present for the schema tool; if it's not
# present, then download it first
if params.hive_jdbc_driver in params.hive_jdbc_drivers_list:
target_directory = format("{params.install_dir}/lib")
# download it if it does not exist
if not os.path.exists(params.source_jdbc_file):
jdbc_connector(params.hive_jdbc_target,
params.hive_previous_jdbc_jar)
target_directory_and_filename = os.path.join(
target_directory, os.path.basename(params.source_jdbc_file))
# copy the JDBC driver from the older metastore location to the new location only
# if it does not already exist
if not os.path.exists(target_directory_and_filename):
Execute(('cp', params.source_jdbc_file, target_directory),
path=["/bin", "/usr/bin/"],
sudo=True)
File(target_directory_and_filename, mode=0644)
# build the schema tool command
binary = format("{hive_schematool_ver_bin}/schematool")
schematool_hive_server_conf_dir = params.hive_server_conf_dir
env_dict = {'HIVE_CONF_DIR': schematool_hive_server_conf_dir}
command = format(
"{binary} -dbType {hive_metastore_db_type} -upgradeSchema")
Execute(command,
user=params.hive_user,
tries=1,
environment=env_dict,
logoutput=True)
def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
expectations = {
"oozie-site":
build_expectations(
'oozie-site', {
"oozie.authentication.type":
"kerberos",
"oozie.service.AuthorizationService.security.enabled":
"true",
"oozie.service.HadoopAccessorService.kerberos.enabled":
"true"
}, [
"local.realm",
"oozie.authentication.kerberos.principal",
"oozie.authentication.kerberos.keytab",
"oozie.service.HadoopAccessorService.kerberos.principal",
"oozie.service.HadoopAccessorService.keytab.file"
], None)
}
security_params = get_params_from_filesystem(
status_params.conf_dir, {'oozie-site.xml': FILE_TYPE_XML})
result_issues = validate_security_config_properties(
security_params, expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if ('oozie-site' not in security_params
or 'oozie.authentication.kerberos.principal'
not in security_params['oozie-site']
or 'oozie.authentication.kerberos.keytab'
not in security_params['oozie-site'] or
'oozie.service.HadoopAccessorService.kerberos.principal'
not in security_params['oozie-site'] or
'oozie.service.HadoopAccessorService.keytab.file'
not in security_params['oozie-site']):
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out({
"securityIssuesFound":
"Keytab file or principal are not set property."
})
return
cached_kinit_executor(
status_params.kinit_path_local,
status_params.oozie_user, security_params['oozie-site']
['oozie.authentication.kerberos.keytab'],
security_params['oozie-site']
['oozie.authentication.kerberos.principal'],
status_params.hostname, status_params.tmp_dir)
cached_kinit_executor(
status_params.kinit_path_local,
status_params.oozie_user, security_params['oozie-site']
['oozie.service.HadoopAccessorService.keytab.file'],
security_params['oozie-site']
['oozie.service.HadoopAccessorService.kerberos.principal'],
status_params.hostname, status_params.tmp_dir)
self.put_structured_out(
{"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append(
"Configuration file %s did not pass the validation. Reason: %s"
% (cf, result_issues[cf]))
self.put_structured_out(
{"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
def upgrade_schema(self, env):
"""
Executes the schema upgrade binary. This is its own function because it could
be called as a standalone task from the upgrade pack, but is safe to run it for each
metastore instance. The schema upgrade on an already upgraded metastore is a NOOP.
The metastore schema upgrade requires a database driver library for most
databases. During an upgrade, it's possible that the library is not present,
so this will also attempt to copy/download the appropriate driver.
This function will also ensure that configurations are written out to disk before running
since the new configs will most likely not yet exist on an upgrade.
Should not be invoked for a DOWNGRADE; Metastore only supports schema upgrades.
"""
Logger.info("Upgrading Hive Metastore Schema")
import status_params
import params
env.set_params(params)
# ensure that configurations are written out before trying to upgrade the schema
# since the schematool needs configs and doesn't know how to use the hive conf override
self.configure(env)
if params.security_enabled:
cached_kinit_executor(status_params.kinit_path_local,
status_params.hive_user,
params.hive_metastore_keytab_path,
params.hive_metastore_principal,
status_params.hostname,
status_params.tmp_dir)
# ensure that the JDBC drive is present for the schema tool; if it's not
# present, then download it first
if params.hive_jdbc_driver in params.hive_jdbc_drivers_list:
target_directory = format("{stack_root}/{version}/hive/lib")
# download it if it does not exist
if not os.path.exists(params.source_jdbc_file):
jdbc_connector(params.hive_jdbc_target,
params.hive_previous_jdbc_jar)
target_directory_and_filename = os.path.join(
target_directory, os.path.basename(params.source_jdbc_file))
if params.sqla_db_used:
target_native_libs_directory = format(
"{target_directory}/native/lib64")
Execute(
format(
"yes | {sudo} cp {jars_in_hive_lib} {target_directory}"
))
Directory(target_native_libs_directory, create_parents=True)
Execute(
format(
"yes | {sudo} cp {libs_in_hive_lib} {target_native_libs_directory}"
))
Execute(
format(
"{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*"
))
else:
# copy the JDBC driver from the older metastore location to the new location only
# if it does not already exist
if not os.path.exists(target_directory_and_filename):
Execute(('cp', params.source_jdbc_file, target_directory),
path=["/bin", "/usr/bin/"],
sudo=True)
File(target_directory_and_filename, mode=0644)
# build the schema tool command
binary = format("{hive_schematool_ver_bin}/schematool")
# the conf.server directory changed locations between stack versions
# since the configurations have not been written out yet during an upgrade
# we need to choose the original legacy location
schematool_hive_server_conf_dir = params.hive_server_conf_dir
if params.current_version is not None:
current_version = format_stack_version(params.current_version)
if not (check_stack_feature(StackFeature.CONFIG_VERSIONING,
current_version)):
schematool_hive_server_conf_dir = LEGACY_HIVE_SERVER_CONF
env_dict = {'HIVE_CONF_DIR': schematool_hive_server_conf_dir}
command = format(
"{binary} -dbType {hive_metastore_db_type} -upgradeSchema")
Execute(command,
user=params.hive_user,
tries=1,
environment=env_dict,
logoutput=True)
def security_status(self, env):
import status_params
env.set_params(status_params)
if status_params.security_enabled:
expectations = {
"oozie-site":
build_expectations('oozie-site',
{
"oozie.authentication.type": "kerberos",
"oozie.service.AuthorizationService.security.enabled": "true",
"oozie.service.HadoopAccessorService.kerberos.enabled": "true"
},
[
"local.realm",
"oozie.authentication.kerberos.principal",
"oozie.authentication.kerberos.keytab",
"oozie.service.HadoopAccessorService.kerberos.principal",
"oozie.service.HadoopAccessorService.keytab.file"
],
None)
}
security_params = get_params_from_filesystem(status_params.conf_dir,
{'oozie-site.xml': FILE_TYPE_XML})
result_issues = validate_security_config_properties(security_params, expectations)
if not result_issues: # If all validations passed successfully
try:
# Double check the dict before calling execute
if ('oozie-site' not in security_params
or 'oozie.authentication.kerberos.principal' not in security_params['oozie-site']
or 'oozie.authentication.kerberos.keytab' not in security_params['oozie-site']
or 'oozie.service.HadoopAccessorService.kerberos.principal' not in security_params['oozie-site']
or 'oozie.service.HadoopAccessorService.keytab.file' not in security_params['oozie-site']):
self.put_structured_out({"securityState": "UNSECURED"})
self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."})
return
cached_kinit_executor(status_params.kinit_path_local,
status_params.oozie_user,
security_params['oozie-site']['oozie.authentication.kerberos.keytab'],
security_params['oozie-site']['oozie.authentication.kerberos.principal'],
status_params.hostname,
status_params.tmp_dir)
cached_kinit_executor(status_params.kinit_path_local,
status_params.oozie_user,
security_params['oozie-site']['oozie.service.HadoopAccessorService.keytab.file'],
security_params['oozie-site']['oozie.service.HadoopAccessorService.kerberos.principal'],
status_params.hostname,
status_params.tmp_dir)
self.put_structured_out({"securityState": "SECURED_KERBEROS"})
except Exception as e:
self.put_structured_out({"securityState": "ERROR"})
self.put_structured_out({"securityStateErrorInfo": str(e)})
else:
issues = []
for cf in result_issues:
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
else:
self.put_structured_out({"securityState": "UNSECURED"})
