hive_metastore_site_supported = True
execute_path = os.environ['PATH'] + os.pathsep + hive_bin + os.pathsep + hadoop_bin_dir
hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName']
hive_jdbc_connection_url = config['configurations']['hive-site']['javax.jdo.option.ConnectionURL']
jdk_location = config['hostLevelParams']['jdk_location']
if credential_store_enabled:
if 'hadoop.security.credential.provider.path' in config['configurations']['hive-site']:
cs_lib_path = config['configurations']['hive-site']['credentialStoreClassPath']
java_home = config['hostLevelParams']['java_home']
alias = 'javax.jdo.option.ConnectionPassword'
provider_path = config['configurations']['hive-site']['hadoop.security.credential.provider.path']
hive_metastore_user_passwd = PasswordString(get_password_from_credential_store(alias, provider_path, cs_lib_path, java_home, jdk_location))
else:
raise Exception("hadoop.security.credential.provider.path property should be set")
else:
hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword']
hive_metastore_user_passwd = unicode(hive_metastore_user_passwd) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd
hive_metastore_db_type = config['configurations']['hive-env']['hive_database_type']
#HACK Temporarily use dbType=azuredb while invoking schematool
if hive_metastore_db_type == "mssql":
hive_metastore_db_type = "azuredb"
#users
hive_user = config['configurations']['hive-env']['hive_user']
# is it a restart command
def ranger_credential_helper(lib_path, alias_key, alias_value, file_path):
import params
java_bin = format('{java_home}/bin/java')
file_path = format('jceks://file{file_path}')
cmd = (java_bin, '-cp', lib_path, 'org.apache.ranger.credentialapi.buildks', 'create', alias_key, '-value', PasswordString(alias_value), '-provider', file_path)
Execute(cmd, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
def setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, hdp_version, credential_file, xa_audit_db_password,
ssl_truststore_password, ssl_keystore_password, component_user, component_group, java_home):
cred_lib_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install/lib/*')
cred_setup_prefix = (format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/ranger_credential_helper.py'), '-l', cred_lib_path)
if audit_db_is_enabled:
cred_setup = cred_setup_prefix + ('-f', credential_file, '-k', 'auditDBCred', '-v', PasswordString(xa_audit_db_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True, sudo=True)
cred_setup = cred_setup_prefix + ('-f', credential_file, '-k', 'sslKeyStore', '-v', PasswordString(ssl_keystore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True, sudo=True)
cred_setup = cred_setup_prefix + ('-f', credential_file, '-k', 'sslTrustStore', '-v', PasswordString(ssl_truststore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True, sudo=True)
File(credential_file,
owner = component_user,
group = component_group,
mode = 0640
)
def enable_kms_plugin():
import params
if params.has_ranger_admin:
ranger_flag = False
if params.stack_supports_ranger_kerberos and params.security_enabled:
if not is_empty(params.rangerkms_principal) and params.rangerkms_principal != '':
ranger_flag = check_ranger_service_support_kerberos(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal)
else:
ranger_flag = check_ranger_service_support_kerberos(params.kms_user, params.spengo_keytab, params.spnego_principal)
else:
ranger_flag = check_ranger_service()
if not ranger_flag:
Logger.error('Error in Get/Create service for Ranger Kms.')
current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
File(format('{kms_conf_dir}/ranger-security.xml'),
owner = params.kms_user,
group = params.kms_group,
mode = 0644,
content = format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>')
)
Directory([os.path.join('/etc', 'ranger', params.repo_name), os.path.join('/etc', 'ranger', params.repo_name, 'policycache')],
owner = params.kms_user,
group = params.kms_group,
mode=0775,
create_parents = True
)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',format('kms_{repo_name}.json')),
owner = params.kms_user,
group = params.kms_group,
mode = 0644
)
# remove plain-text password from xml configs
plugin_audit_properties_copy = {}
plugin_audit_properties_copy.update(params.config['configurations']['ranger-kms-audit'])
if params.plugin_audit_password_property in plugin_audit_properties_copy:
plugin_audit_properties_copy[params.plugin_audit_password_property] = "crypted"
XmlConfig("ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=plugin_audit_properties_copy,
configuration_attributes=params.config['configuration_attributes']['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig("ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-security'],
configuration_attributes=params.config['configuration_attributes']['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
# remove plain-text password from xml configs
ranger_kms_policymgr_ssl_copy = {}
ranger_kms_policymgr_ssl_copy.update(params.config['configurations']['ranger-kms-policymgr-ssl'])
for prop in params.kms_plugin_password_properties:
if prop in ranger_kms_policymgr_ssl_copy:
ranger_kms_policymgr_ssl_copy[prop] = "crypted"
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=ranger_kms_policymgr_ssl_copy,
configuration_attributes=params.config['configuration_attributes']['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'auditDBCred', '-v', PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslKeyStore', '-v', PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslTrustStore', '-v', PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
File(params.credential_file,
owner = params.kms_user,
group = params.kms_group,
mode = 0640
)
# create ranger kms audit directory
if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True
)
params.HdfsResource("/ranger/audit/kms",
type="directory",
action="create_on_execute",
owner=params.kms_user,
group=params.kms_group,
mode=0750,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
if params.xa_audit_hdfs_is_enabled and len(params.namenode_host) > 1:
Logger.info('Audit to Hdfs enabled in NameNode HA environment, creating hdfs-site.xml')
XmlConfig("hdfs-site.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644
)
else:
File(format('{kms_conf_dir}/hdfs-site.xml'), action="delete")
def enable_kms_plugin():
import params
if params.has_ranger_admin:
if params.stack_supports_ranger_kerberos and params.security_enabled:
ranger_flag = check_ranger_service_support_kerberos()
else:
ranger_flag = check_ranger_service()
if not ranger_flag:
Logger.error('Error in Get/Create service for Ranger Kms.')
current_datetime = datetime.now()
File(
format('{kms_conf_dir}/ranger-security.xml'),
owner=params.kms_user,
group=params.kms_group,
mode=0644,
content=InlineTemplate(
format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
)))
Directory([
os.path.join('/etc', 'ranger', params.repo_name),
os.path.join('/etc', 'ranger', params.repo_name, 'policycache')
],
owner=params.kms_user,
group=params.kms_group,
mode=0775,
create_parents=True)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',
format('kms_{repo_name}.json')),
owner=params.kms_user,
group=params.kms_group,
mode=0644)
XmlConfig(
"ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-audit'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-security'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-policymgr-ssl'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'auditDBCred', '-v',
PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.credential_file,
owner=params.kms_user,
group=params.kms_group,
mode=0640)
def do_keystore_setup(cred_provider_path, credential_alias, credential_password):
import params
if cred_provider_path is not None:
cred_setup = params.cred_setup_prefix + ('-f', cred_provider_path, '-k', credential_alias, '-v', PasswordString(credential_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True,
)
File(cred_provider_path,
owner = params.kms_user,
group = params.kms_group,
mode = 0640
)
def encrypt_sensitive_properties(
nifi_config_dir,
jdk64_home,
java_options,
nifi_user,
last_master_key,
master_key_password,
is_starting,
toolkit_tmp_dir,
stack_version_buildnum,
service,
nifi_flow_config_dir=None,
nifi_sensitive_props_key=None,
support_encrypt_authorizers=None,
):
encrypt_config_script = get_toolkit_script('encrypt-config.sh',
toolkit_tmp_dir,
stack_version_buildnum)
encrypt_config_command = (encrypt_config_script, )
environment = {'JAVA_HOME': jdk64_home, 'JAVA_OPTS': java_options}
File(encrypt_config_script, mode=0755)
if is_starting:
if service == NIFI:
Logger.info("Encrypting NiFi sensitive configuration properties")
encrypt_config_command += ('-v', '-b',
nifi_config_dir + '/bootstrap.conf')
encrypt_config_command += ('-n',
nifi_config_dir + '/nifi.properties')
if (sudo.path_isfile(nifi_flow_config_dir + '/flow.xml.gz') and
len(sudo.read_file(nifi_flow_config_dir + '/flow.xml.gz'))
> 0):
encrypt_config_command += (
'-f', nifi_flow_config_dir + '/flow.xml.gz', '-s',
PasswordString(nifi_sensitive_props_key))
if contains_providers(
nifi_config_dir + '/login-identity-providers.xml',
"provider"):
encrypt_config_command += ('-l', nifi_config_dir +
'/login-identity-providers.xml')
if support_encrypt_authorizers and contains_providers(
nifi_config_dir + '/authorizers.xml', "authorizer"):
encrypt_config_command += ('-a', nifi_config_dir +
'/authorizers.xml')
if last_master_key:
encrypt_config_command += ('-m', '-e',
PasswordString(last_master_key))
encrypt_config_command += ('-p',
PasswordString(master_key_password))
Execute(encrypt_config_command,
user=nifi_user,
logoutput=False,
environment=environment)
elif service == NIFI_REGISTRY:
Logger.info(
"Encrypting NiFi Registry sensitive configuration properties")
encrypt_config_command += ('--nifiRegistry', '-v', '-b',
nifi_config_dir + '/bootstrap.conf')
encrypt_config_command += ('-r', nifi_config_dir +
'/nifi-registry.properties')
if contains_providers(nifi_config_dir + '/identity-providers.xml',
"provider"):
encrypt_config_command += ('-i', nifi_config_dir +
'/identity-providers.xml')
if contains_providers(nifi_config_dir + '/authorizers.xml',
"authorizer"):
encrypt_config_command += ('-a', nifi_config_dir +
'/authorizers.xml')
if last_master_key:
encrypt_config_command += ('--oldKey',
PasswordString(last_master_key))
encrypt_config_command += ('-p',
PasswordString(master_key_password))
Execute(encrypt_config_command,
user=nifi_user,
logoutput=False,
environment=environment)
cmd = (java_bin, '-cp', cs_lib_path, credential_util_cmd, 'get', alias,
'-provider', provider_path)
cmd_result, std_out_msg = checked_call(cmd)
if cmd_result != 0:
message = 'The following error occurred while executing {0}: {1}'.format(
' '.join(cmd), std_out_msg)
Logger.error(message)
raise
std_out_lines = std_out_msg.split('\n')
passwd = std_out_lines[
-1] # Get the last line of the output, to skip warnings if any.
return passwd
if credential_store_enabled:
hive_metastore_user_passwd = PasswordString(getHiveMetastorePassword())
else:
hive_metastore_user_passwd = config['configurations']['hive-site'][
'javax.jdo.option.ConnectionPassword']
hive_metastore_user_passwd = unicode(
hive_metastore_user_passwd
) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd
hive_metastore_db_type = config['configurations']['hive-env'][
'hive_database_type']
#HACK Temporarily use dbType=azuredb while invoking schematool
if hive_metastore_db_type == "mssql":
hive_metastore_db_type = "azuredb"
#users
hive_user = config['configurations']['hive-env']['hive_user']
def do_keystore_setup(rolling_upgrade=False):
import params
ranger_home = params.ranger_home
cred_lib_path = params.cred_lib_path
cred_setup_prefix = params.cred_setup_prefix
if rolling_upgrade:
ranger_home = format("/usr/hdp/{version}/ranger-admin")
cred_lib_path = os.path.join(ranger_home,"cred","lib","*")
cred_setup_prefix = (format('{ranger_home}/ranger_credential_helper.py'), '-l', cred_lib_path)
if not is_empty(params.ranger_credential_provider_path):
jceks_path = params.ranger_credential_provider_path
cred_setup = cred_setup_prefix + ('-f', jceks_path, '-k', params.ranger_jpa_jdbc_credential_alias, '-v', PasswordString(params.ranger_ambari_db_password), '-c', '1')
Execute(cred_setup,
environment={'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True
)
File(params.ranger_credential_provider_path,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
if not is_empty(params.ranger_credential_provider_path) and (params.ranger_audit_source_type).lower() == 'db' and not is_empty(params.ranger_ambari_audit_db_password):
jceks_path = params.ranger_credential_provider_path
cred_setup = cred_setup_prefix + ('-f', jceks_path, '-k', params.ranger_jpa_audit_jdbc_credential_alias, '-v', PasswordString(params.ranger_ambari_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True
)
File(params.ranger_credential_provider_path,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
def enable_kms_plugin():
import params
if params.has_ranger_admin:
ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url)
response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(
params.policymgr_mgr_url + '/login.jsp', 'test:test')
if response_code is not None and response_code == 200:
ambari_ranger_admin, ambari_ranger_password = ranger_adm_obj.create_ambari_admin_user(
params.ambari_ranger_admin, params.ambari_ranger_password,
params.admin_uname_password)
ambari_username_password_for_ranger = ambari_ranger_admin + ':' + ambari_ranger_password
else:
raise Fail('Ranger service is not started on given host')
if ambari_ranger_admin != '' and ambari_ranger_password != '':
get_repo_flag = get_repo(params.policymgr_mgr_url,
params.repo_name,
ambari_username_password_for_ranger)
if not get_repo_flag:
create_repo(params.policymgr_mgr_url,
json.dumps(params.kms_ranger_plugin_repo),
ambari_username_password_for_ranger)
else:
raise Fail('Ambari admin username and password not available')
current_datetime = datetime.now()
File(
format('{kms_conf_dir}/ranger-security.xml'),
owner=params.kms_user,
group=params.kms_group,
mode=0644,
content=InlineTemplate(
format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
)))
Directory([
os.path.join('/etc', 'ranger', params.repo_name),
os.path.join('/etc', 'ranger', params.repo_name, 'policycache')
],
owner=params.kms_user,
group=params.kms_group,
mode=0775,
recursive=True)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',
format('kms_{repo_name}.json')),
owner=params.kms_user,
group=params.kms_group,
mode=0644)
XmlConfig(
"ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-audit'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-security'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-policymgr-ssl'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'auditDBCred', '-v',
PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.credential_file,
owner=params.kms_user,
group=params.kms_group,
mode=0640)
def setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, stack_version, credential_file, xa_audit_db_password,
ssl_truststore_password, ssl_keystore_password, component_user, component_group, java_home, cred_lib_path_override = None, cred_setup_prefix_override = None):
stack_root = Script.get_stack_root()
service_name = str(service_name).lower()
if cred_lib_path_override is not None:
cred_lib_path = cred_lib_path_override
else:
cred_lib_path = format('{stack_root}/{stack_version}/ranger-{service_name}-plugin/install/lib/*')
if cred_setup_prefix_override is not None:
cred_setup_prefix = cred_setup_prefix_override
else:
cred_setup_prefix = (format('{stack_root}/{stack_version}/ranger-{service_name}-plugin/ranger_credential_helper.py'), '-l', cred_lib_path)
if audit_db_is_enabled:
cred_setup = cred_setup_prefix + ('-f', credential_file, '-k', 'auditDBCred', '-v', PasswordString(xa_audit_db_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True, sudo=True)
cred_setup = cred_setup_prefix + ('-f', credential_file, '-k', 'sslKeyStore', '-v', PasswordString(ssl_keystore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True, sudo=True)
cred_setup = cred_setup_prefix + ('-f', credential_file, '-k', 'sslTrustStore', '-v', PasswordString(ssl_truststore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True, sudo=True)
File(credential_file,
owner = component_user,
group = component_group,
mode = 0640
)
def log_str(self, key, value):
# Hide the passwords from text representations
return repr(PasswordString(value))
def create_password_in_credential_store(alias, provider_path, cs_lib_path, java_home, jdk_location, password):
downloadjar(cs_lib_path, jdk_location)
#Execute the creation and overwrite password
java_bin = '{java_home}/bin/java'.format(java_home=java_home)
cmd = (java_bin, '-cp', cs_lib_path, credential_util_cmd, 'create', alias, '-value', PasswordString(password) ,'-provider', provider_path, '-f')
Execute(cmd)
def setup_tagsync(upgrade_type=None):
import params
ranger_tagsync_home = params.ranger_tagsync_home
ranger_home = params.ranger_home
ranger_tagsync_conf = params.ranger_tagsync_conf
tagsync_log4j_file = format('{ranger_tagsync_conf}/log4j.xml')
tagsync_services_file = format(
'{ranger_tagsync_home}/ranger-tagsync-services.sh')
Directory(format("{ranger_tagsync_conf}"),
owner=params.unix_user,
group=params.unix_group,
create_parents=True)
Directory(
params.ranger_pid_dir,
mode=0750,
create_parents=True,
owner=params.unix_user,
group=params.unix_group,
cd_access="a",
)
Directory(params.tagsync_log_dir,
create_parents=True,
owner=params.unix_user,
group=params.unix_group,
cd_access="a",
mode=0755)
File(format('{ranger_tagsync_conf}/ranger-tagsync-env-logdir.sh'),
content=format("export RANGER_TAGSYNC_LOG_DIR={tagsync_log_dir}"),
owner=params.unix_user,
group=params.unix_group,
mode=0755)
XmlConfig(
"ranger-tagsync-site.xml",
conf_dir=ranger_tagsync_conf,
configurations=params.config['configurations']['ranger-tagsync-site'],
configuration_attributes=params.config['configuration_attributes']
['ranger-tagsync-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
PropertiesFile(format('{ranger_tagsync_conf}/application.properties'),
properties=params.tagsync_application_properties,
mode=0755,
owner=params.unix_user,
group=params.unix_group)
if upgrade_type is not None:
src_file = format(
'{ranger_tagsync_home}/ews/webapp/WEB-INF/classes/conf.dist/log4j.xml'
)
dst_file = format('{tagsync_log4j_file}')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
if os.path.isfile(tagsync_log4j_file):
File(tagsync_log4j_file,
owner=params.unix_user,
group=params.unix_group)
else:
Logger.warning(
'Required file {0} does not exist, copying the file to {1} path'.
format(tagsync_log4j_file, ranger_tagsync_conf))
src_file = format(
'{ranger_tagsync_home}/ews/webapp/WEB-INF/classes/conf.dist/log4j.xml'
)
dst_file = format('{tagsync_log4j_file}')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
File(tagsync_log4j_file,
owner=params.unix_user,
group=params.unix_group)
cred_file = format('{ranger_home}/ranger_credential_helper.py')
if os.path.isfile(
format('{ranger_tagsync_home}/ranger_credential_helper.py')):
cred_file = format('{ranger_tagsync_home}/ranger_credential_helper.py')
cred_lib = os.path.join(ranger_tagsync_home, "lib", "*")
cred_setup_prefix = (cred_file, '-l', cred_lib)
if not is_empty(params.tagsync_jceks_path) and not is_empty(
params.ranger_tagsync_tagadmin_password
) and params.tagsync_enabled:
cred_setup = cred_setup_prefix + (
'-f', params.tagsync_jceks_path, '-k', 'tagadmin.user.password',
'-v', PasswordString(
params.ranger_tagsync_tagadmin_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.tagsync_jceks_path,
owner=params.unix_user,
group=params.unix_group,
mode=0640)
if os.path.isfile(tagsync_services_file):
File(
tagsync_services_file,
mode=0755,
)
Execute(('ln', '-sf', format('{tagsync_services_file}'),
'/usr/bin/ranger-tagsync'),
not_if=format("ls /usr/bin/ranger-tagsync"),
only_if=format("ls {tagsync_services_file}"),
sudo=True)
def setup_usersync(upgrade_type=None):
import params
usersync_home = params.usersync_home
ranger_home = params.ranger_home
ranger_ugsync_conf = params.ranger_ugsync_conf
if not is_empty(
params.ranger_usersync_ldap_ldapbindpassword
) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder':
password_validation(params.ranger_usersync_ldap_ldapbindpassword)
if upgrade_type is not None:
usersync_home = format("/usr/hdp/{version}/ranger-usersync")
ranger_home = format("/usr/hdp/{version}/ranger-admin")
ranger_ugsync_conf = format("/usr/hdp/{version}/ranger-usersync/conf")
Directory(params.ranger_pid_dir,
mode=0750,
owner=params.unix_user,
group=params.unix_group)
Directory(params.usersync_log_dir,
owner=params.unix_user,
group=params.unix_group)
Directory(format("{ranger_ugsync_conf}/"), owner=params.unix_user)
if upgrade_type is not None:
src_file = format(
'{usersync_home}/conf.dist/ranger-ugsync-default.xml')
dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
src_file = format('{usersync_home}/conf.dist/log4j.xml')
dst_file = format('{usersync_home}/conf/log4j.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
XmlConfig(
"ranger-ugsync-site.xml",
conf_dir=ranger_ugsync_conf,
configurations=params.config['configurations']['ranger-ugsync-site'],
configuration_attributes=params.config['configuration_attributes']
['ranger-ugsync-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
if os.path.isfile(params.ranger_ugsync_default_file):
File(params.ranger_ugsync_default_file,
owner=params.unix_user,
group=params.unix_group)
if os.path.isfile(params.usgsync_log4j_file):
File(params.usgsync_log4j_file,
owner=params.unix_user,
group=params.unix_group)
if os.path.isfile(params.cred_validator_file):
File(params.cred_validator_file, group=params.unix_group, mode=04555)
cred_file = format('{ranger_home}/ranger_credential_helper.py')
if os.path.isfile(format('{usersync_home}/ranger_credential_helper.py')):
cred_file = format('{usersync_home}/ranger_credential_helper.py')
cred_lib = os.path.join(usersync_home, "lib", "*")
cred_setup_prefix = (cred_file, '-l', cred_lib)
cred_setup = cred_setup_prefix + (
'-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.key.password',
'-v', PasswordString(
params.ranger_usersync_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', params.ugsync_jceks_path, '-k', 'ranger.usersync.ldap.bindalias',
'-v', PasswordString(
params.ranger_usersync_ldap_ldapbindpassword), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', params.ugsync_jceks_path, '-k',
'usersync.ssl.truststore.password', '-v',
PasswordString(params.ranger_usersync_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.ugsync_jceks_path,
owner=params.unix_user,
group=params.unix_group,
mode=0640)
File([params.usersync_start, params.usersync_stop],
owner=params.unix_user,
group=params.unix_group)
File(
params.usersync_services_file,
mode=0755,
)
Execute(('ln', '-sf', format('{usersync_services_file}'),
'/usr/bin/ranger-usersync'),
not_if=format("ls /usr/bin/ranger-usersync"),
only_if=format("ls {usersync_services_file}"),
sudo=True)
if not os.path.isfile(params.ranger_usersync_keystore_file):
cmd = format(
"{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'"
)
Execute(cmd, logoutput=True, user=params.unix_user)
File(params.ranger_usersync_keystore_file,
owner=params.unix_user,
group=params.unix_group,
mode=0640)
def setup_usersync():
import params
Directory(params.ranger_pid_dir,
mode=0750,
owner = params.unix_user,
group = params.unix_group
)
Directory(params.usersync_log_dir,
owner = params.unix_user,
group = params.unix_group
)
Directory(format("{ranger_ugsync_conf}/"),
owner = params.unix_user
)
XmlConfig("ranger-ugsync-site.xml",
conf_dir=params.ranger_ugsync_conf,
configurations=params.config['configurations']['ranger-ugsync-site'],
configuration_attributes=params.config['configuration_attributes']['ranger-ugsync-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
cred_lib = os.path.join(params.usersync_home,"lib","*")
cred_setup_prefix = (format('{ranger_home}/ranger_credential_helper.py'), '-l', cred_lib)
cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.key.password', '-v', PasswordString(params.ranger_usersync_keystore_password), '-c', '1')
Execute(cred_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'ranger.usersync.ldap.bindalias', '-v', PasswordString(params.ranger_usersync_ldap_ldapbindpassword), '-c', '1')
Execute(cred_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.truststore.password', '-v', PasswordString(params.ranger_usersync_truststore_password), '-c', '1')
Execute(cred_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
File(params.ugsync_jceks_path,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
File([params.usersync_start, params.usersync_stop],
owner = params.unix_user,
group = params.unix_group
)
File(params.usersync_services_file,
mode = 0755,
)
Execute(('ln','-sf', format('{usersync_services_file}'),'/usr/bin/ranger-usersync'),
not_if=format("ls /usr/bin/ranger-usersync"),
only_if=format("ls {usersync_services_file}"),
sudo=True)
if not os.path.isfile(params.ranger_usersync_keystore_file):
cmd = format("{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'")
Execute(cmd, logoutput=True, user = params.unix_user)
File(params.ranger_usersync_keystore_file,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
def generateJceks(self, commandJson):
"""
Generates the JCEKS file with passwords for the service specified in commandJson
:param commandJson: command JSON
:return: An exit value from the external process that generated the JCEKS file. None if
there are no passwords in the JSON.
"""
cmd_result = None
roleCommand = None
if 'roleCommand' in commandJson:
roleCommand = commandJson['roleCommand']
task_id = None
if 'taskId' in commandJson:
task_id = commandJson['taskId']
logger.info(
'Generating the JCEKS file: roleCommand={0} and taskId = {1}'.
format(roleCommand, task_id))
# Set up the variables for the external command to generate a JCEKS file
java_home = commandJson['hostLevelParams']['java_home']
java_bin = '{java_home}/bin/java'.format(java_home=java_home)
cs_lib_path = self.credential_shell_lib_path
serviceName = commandJson['serviceName']
# Gather the password values and remove them from the configuration
configtype_credentials = self.getConfigTypeCredentials(commandJson)
# CS is enabled but no config property is available for this command
if len(configtype_credentials) == 0:
logger.info(
"Credential store is enabled but no property are found that can be encrypted."
)
commandJson['credentialStoreEnabled'] = "false"
for config_type, credentials in configtype_credentials.items():
config = commandJson['configurations'][config_type]
file_path = os.path.join(self.getProviderDirectory(serviceName),
"{0}.jceks".format(config_type))
if os.path.exists(file_path):
os.remove(file_path)
provider_path = 'jceks://file{file_path}'.format(
file_path=file_path)
logger.info('provider_path={0}'.format(provider_path))
for alias, pwd in credentials.items():
logger.debug("config={0}".format(config))
protected_pwd = PasswordString(pwd)
# Generate the JCEKS file
cmd = (java_bin, '-cp', cs_lib_path, self.credential_shell_cmd,
'create', alias, '-value', protected_pwd, '-provider',
provider_path)
logger.info(cmd)
cmd_result = subprocess.call(cmd)
logger.info('cmd_result = {0}'.format(cmd_result))
os.chmod(
file_path, 0644
) # group and others should have read access so that the service user can read
# Add JCEKS provider path instead
config[self.CREDENTIAL_PROVIDER_PROPERTY_NAME] = provider_path
config[self.CREDENTIAL_STORE_CLASS_PATH_NAME] = cs_lib_path
return cmd_result
def do_keystore_setup(cred_provider_path, credential_alias, credential_password):
import params
if cred_provider_path is not None:
java_bin = format('{java_home}/bin/java')
file_path = format('jceks://file{cred_provider_path}')
cmd = (java_bin, '-cp', params.cred_lib_path, 'org.apache.ranger.credentialapi.buildks', 'create', credential_alias, '-value', PasswordString(credential_password), '-provider', file_path)
Execute(cmd,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True,
)
File(cred_provider_path,
owner = params.kms_user,
group = params.kms_group,
mode = 0640
)
def enable_kms_plugin():
import params
if params.has_ranger_admin:
count = 0
while count < 5:
ranger_flag = check_ranger_service()
if ranger_flag:
break
else:
time.sleep(5) # delay for 5 seconds
count = count + 1
else:
Logger.error("Ranger service is not reachable after {0} tries".format(count))
current_datetime = datetime.now()
File(format('{kms_conf_dir}/ranger-security.xml'),
owner = params.kms_user,
group = params.kms_group,
mode = 0644,
content = InlineTemplate(format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'))
)
Directory([os.path.join('/etc', 'ranger', params.repo_name), os.path.join('/etc', 'ranger', params.repo_name, 'policycache')],
owner = params.kms_user,
group = params.kms_group,
mode=0775,
recursive = True
)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',format('kms_{repo_name}.json')),
owner = params.kms_user,
group = params.kms_group,
mode = 0644
)
XmlConfig("ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-audit'],
configuration_attributes=params.config['configuration_attributes']['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig("ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-security'],
configuration_attributes=params.config['configuration_attributes']['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-policymgr-ssl'],
configuration_attributes=params.config['configuration_attributes']['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'auditDBCred', '-v', PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslKeyStore', '-v', PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslTrustStore', '-v', PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
File(params.credential_file,
owner = params.kms_user,
group = params.kms_group,
mode = 0640
)
def encrypt_sensitive_properties(config_version_file, current_version,
nifi_config_dir, jdk64_home, java_options,
nifi_user, nifi_group, master_key_password,
nifi_flow_config_dir,
nifi_sensitive_props_key, is_starting,
toolkit_tmp_dir, support_encrypt_authorizers):
Logger.info("Encrypting NiFi sensitive configuration properties")
encrypt_config_script = get_toolkit_script('encrypt-config.sh',
toolkit_tmp_dir)
encrypt_config_command = (encrypt_config_script, )
environment = {'JAVA_HOME': jdk64_home}
if java_options:
environment['JAVA_OPTS'] = java_options
File(encrypt_config_script, mode=0755)
if is_starting:
last_master_key_password = None
last_config_version = get_config_version(config_version_file,
'encrypt')
encrypt_config_command += ('-v', '-b',
nifi_config_dir + '/bootstrap.conf')
encrypt_config_command += ('-n', nifi_config_dir + '/nifi.properties')
if (sudo.path_isfile(nifi_flow_config_dir + '/flow.xml.gz') and len(
sudo.read_file(nifi_flow_config_dir + '/flow.xml.gz')) > 0):
encrypt_config_command += (
'-f', nifi_flow_config_dir + '/flow.xml.gz', '-s',
PasswordString(nifi_sensitive_props_key))
if contains_providers(
nifi_config_dir + '/login-identity-providers.xml', "provider"):
encrypt_config_command += ('-l', nifi_config_dir +
'/login-identity-providers.xml')
if support_encrypt_authorizers and contains_providers(
nifi_config_dir + '/authorizers.xml', "authorizer"):
encrypt_config_command += ('-a',
nifi_config_dir + '/authorizers.xml')
if last_config_version:
last_config = get_config_by_version('/var/lib/ambari-agent/data',
'nifi-ambari-config',
last_config_version)
last_master_key_password = last_config['configurations'][
'nifi-ambari-config'][
'nifi.security.encrypt.configuration.password']
if last_master_key_password and last_master_key_password != master_key_password:
encrypt_config_command += (
'-m', '-w', PasswordString(last_master_key_password))
encrypt_config_command += ('-p', PasswordString(master_key_password))
Execute(encrypt_config_command,
user=nifi_user,
logoutput=False,
environment=environment)
save_config_version(config_version_file, 'encrypt', current_version,
nifi_user, nifi_group)
