def setup_ranger_plugin_keystore(service_name,
audit_db_is_enabled,
stack_version,
credential_file,
xa_audit_db_password,
ssl_truststore_password,
ssl_keystore_password,
component_user,
component_group,
java_home,
cred_lib_path_override=None,
cred_setup_prefix_override=None):
stack_root = Script.get_stack_root()
service_name = str(service_name).lower()
if cred_lib_path_override is not None:
cred_lib_path = cred_lib_path_override
else:
cred_lib_path = format(
'{stack_root}/{stack_version}/ranger-{service_name}-plugin/install/lib/*'
)
if cred_setup_prefix_override is not None:
cred_setup_prefix = cred_setup_prefix_override
else:
cred_setup_prefix = (format(
'{stack_root}/{stack_version}/ranger-{service_name}-plugin/ranger_credential_helper.py'
), '-l', cred_lib_path)
if audit_db_is_enabled:
cred_setup = cred_setup_prefix + (
'-f', credential_file, '-k', 'auditDBCred', '-v',
PasswordString(xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': java_home},
logoutput=True,
sudo=True)
File(credential_file,
owner=component_user,
group=component_group,
mode=0640)
def encrypt_sensitive_properties(config_version_file, current_version,
nifi_registry_config_dir, jdk64_home,
java_options, nifi_registry_user,
nifi_registry_group, master_key_password,
is_starting, toolkit_tmp_dir):
Logger.info("Encrypting NiFi Registry sensitive configuration properties")
encrypt_config_script = get_toolkit_script('encrypt-config.sh',
toolkit_tmp_dir)
encrypt_config_command = (encrypt_config_script, )
environment = {'JAVA_HOME': jdk64_home, 'JAVA_OPTS': java_options}
File(encrypt_config_script, mode=0755)
if is_starting:
last_master_key_password = None
last_config_version = get_config_version(config_version_file,
'encrypt')
encrypt_config_command += ('--nifiRegistry', '-v', '-b',
nifi_registry_config_dir +
'/bootstrap.conf')
encrypt_config_command += ('-r', nifi_registry_config_dir +
'/nifi-registry.properties')
if contains_providers(
nifi_registry_config_dir + '/identity-providers.xml',
"provider"):
encrypt_config_command += ('-i', nifi_registry_config_dir +
'/identity-providers.xml')
if contains_providers(nifi_registry_config_dir + '/authorizers.xml',
"authorizer"):
encrypt_config_command += ('-a', nifi_registry_config_dir +
'/authorizers.xml')
if last_config_version:
last_config = get_config_by_version('/var/lib/ambari-agent/data',
'nifi-registry-ambari-config',
last_config_version)
last_master_key_password = last_config['configurations'][
'nifi-registry-ambari-config'][
'nifi.registry.security.encrypt.configuration.password']
if last_master_key_password:
encrypt_config_command += (
'--oldPassword', PasswordString(last_master_key_password))
encrypt_config_command += ('-p', PasswordString(master_key_password))
Execute(encrypt_config_command,
user=nifi_registry_user,
logoutput=False,
environment=environment)
save_config_version(config_version_file, 'encrypt', current_version,
nifi_registry_user, nifi_registry_group)
def do_keystore_setup(rolling_upgrade=False):
import params
ranger_home = params.ranger_home
cred_lib_path = params.cred_lib_path
cred_setup_prefix = params.cred_setup_prefix
if rolling_upgrade:
ranger_home = format("/usr/hdp/{version}/ranger-admin")
cred_lib_path = os.path.join(ranger_home, "cred", "lib", "*")
cred_setup_prefix = (
format('{ranger_home}/ranger_credential_helper.py'), '-l',
cred_lib_path)
if not is_empty(params.ranger_credential_provider_path):
jceks_path = params.ranger_credential_provider_path
cred_setup = cred_setup_prefix + (
'-f', jceks_path, '-k', params.ranger_jpa_jdbc_credential_alias,
'-v', PasswordString(params.ranger_ambari_db_password), '-c', '1')
Execute(cred_setup,
environment={
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home
},
logoutput=True,
sudo=True)
File(params.ranger_credential_provider_path,
owner=params.unix_user,
group=params.unix_group,
mode=0640)
if not is_empty(params.ranger_credential_provider_path) and (
params.ranger_audit_source_type).lower() == 'db' and not is_empty(
params.ranger_ambari_audit_db_password):
jceks_path = params.ranger_credential_provider_path
cred_setup = cred_setup_prefix + (
'-f', jceks_path, '-k',
params.ranger_jpa_audit_jdbc_credential_alias, '-v',
PasswordString(params.ranger_ambari_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home
},
logoutput=True,
sudo=True)
File(params.ranger_credential_provider_path,
owner=params.unix_user,
group=params.unix_group,
mode=0640)
def dump(config_json, config_dict, user, group):
File(config_json,
owner=user,
group=group,
mode=0640,
content=PasswordString(
json.dumps(config_dict, sort_keys=True, indent=4)))
def do_keystore_setup(cred_provider_path, credential_alias,
credential_password):
import params
if cred_provider_path is not None:
java_bin = format('{java_home}/bin/java')
file_path = format('jceks://file{cred_provider_path}')
cmd = (java_bin, '-cp', params.cred_lib_path,
'org.apache.ranger.credentialapi.buildks',
'create', credential_alias, '-value',
PasswordString(credential_password), '-provider', file_path)
Execute(
cmd,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True,
)
File(cred_provider_path,
owner=params.kms_user,
group=params.kms_group,
only_if=format("test -e {cred_provider_path}"),
mode=0640)
dot_jceks_crc_file_path = os.path.join(
os.path.dirname(cred_provider_path),
"." + os.path.basename(cred_provider_path) + ".crc")
File(dot_jceks_crc_file_path,
owner=params.kms_user,
group=params.kms_group,
only_if=format("test -e {dot_jceks_crc_file_path}"),
mode=0640)
def generateJceks(self, commandJson):
"""
Generates the JCEKS file with passwords for the service specified in commandJson
:param commandJson: command JSON
:return: An exit value from the external process that generated the JCEKS file. None if
there are no passwords in the JSON.
"""
cmd_result = None
roleCommand = None
if 'roleCommand' in commandJson:
roleCommand = commandJson['roleCommand']
task_id = None
if 'taskId' in commandJson:
task_id = commandJson['taskId']
logger.info('Generating the JCEKS file: roleCommand={0} and taskId = {1}'.format(roleCommand, task_id))
# Set up the variables for the external command to generate a JCEKS file
java_home = commandJson['hostLevelParams']['java_home']
java_bin = '{java_home}/bin/java'.format(java_home=java_home)
cs_lib_path = self.credential_shell_lib_path
serviceName = commandJson['serviceName']
# Gather the password values and remove them from the configuration
configtype_credentials = self.getConfigTypeCredentials(commandJson)
# CS is enabled but no config property is available for this command
if len(configtype_credentials) == 0:
logger.info("Credential store is enabled but no property are found that can be encrypted.")
commandJson['credentialStoreEnabled'] = "false"
for config_type, credentials in configtype_credentials.items():
config = commandJson['configurations'][config_type]
if 'role' in commandJson and commandJson['role']:
roleName = commandJson['role']
file_path = os.path.join(self.getProviderDirectory(roleName), "{0}.jceks".format(config_type))
else:
file_path = os.path.join(self.getProviderDirectory(serviceName), "{0}.jceks".format(config_type))
if os.path.exists(file_path):
os.remove(file_path)
provider_path = 'jceks://file{file_path}'.format(file_path=file_path)
logger.info('provider_path={0}'.format(provider_path))
for alias, pwd in credentials.items():
logger.debug("config={0}".format(config))
protected_pwd = PasswordString(pwd)
# Generate the JCEKS file
cmd = (java_bin, '-cp', cs_lib_path, self.credential_shell_cmd, 'create',
alias, '-value', protected_pwd, '-provider', provider_path)
logger.info(cmd)
cmd_result = subprocess.call(cmd)
logger.info('cmd_result = {0}'.format(cmd_result))
os.chmod(file_path, 0644) # group and others should have read access so that the service user can read
# Add JCEKS provider path instead
config[self.CREDENTIAL_PROVIDER_PROPERTY_NAME] = provider_path
config[self.CREDENTIAL_STORE_CLASS_PATH_NAME] = cs_lib_path
return cmd_result
def dump(config_json, config_dict, nifi_registry_user, nifi_registry_group):
File(config_json,
owner=nifi_registry_user,
group=nifi_registry_group,
mode=0600,
content=PasswordString(
json.dumps(config_dict, sort_keys=True, indent=4)))
def dump(config_json, config_dict):
import params
File(config_json,
owner=params.nifi_user,
group=params.nifi_group,
mode=0600,
content=PasswordString(
json.dumps(config_dict, sort_keys=True, indent=4)))
def do_keystore_setup(upgrade_type=None):
import params
ranger_home = params.ranger_home
cred_lib_path = params.cred_lib_path
cred_setup_prefix = params.cred_setup_prefix
if not is_empty(params.ranger_credential_provider_path):
jceks_path = params.ranger_credential_provider_path
cred_setup = cred_setup_prefix + (
'-f', jceks_path, '-k', params.ranger_jpa_jdbc_credential_alias,
'-v', PasswordString(params.ranger_ambari_db_password), '-c', '1')
Execute(cred_setup,
environment={
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home
},
logoutput=True,
sudo=True)
File(params.ranger_credential_provider_path,
owner=params.unix_user,
group=params.unix_group,
mode=0640)
if not is_empty(params.ranger_credential_provider_path) and (
params.ranger_audit_source_type).lower() == 'db' and not is_empty(
params.ranger_ambari_audit_db_password):
jceks_path = params.ranger_credential_provider_path
cred_setup = cred_setup_prefix + (
'-f', jceks_path, '-k',
params.ranger_jpa_audit_jdbc_credential_alias, '-v',
PasswordString(params.ranger_ambari_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home
},
logoutput=True,
sudo=True)
File(params.ranger_credential_provider_path,
owner=params.unix_user,
group=params.unix_group,
mode=0640)
def create_password_in_credential_store(alias, provider_path, cs_lib_path,
java_home, jdk_location, password):
downloadjar(cs_lib_path, jdk_location)
#Execute the creation and overwrite password
java_bin = '{java_home}/bin/java'.format(java_home=java_home)
cmd = (java_bin, '-cp', cs_lib_path,
credential_util_cmd, 'create', alias, '-value',
PasswordString(password), '-provider', provider_path, '-f')
Execute(cmd)
def generateJceks(self, commandJson):
"""
Generates the JCEKS file with passwords for the service specified in commandJson
:param commandJson: command JSON
:return: An exit value from the external process that generated the JCEKS file. None if
there are no passwords in the JSON.
"""
cmd_result = None
roleCommand = None
if 'roleCommand' in commandJson:
roleCommand = commandJson['roleCommand']
logger.info('generateJceks: roleCommand={0}'.format(roleCommand))
# Set up the variables for the external command to generate a JCEKS file
java_home = commandJson['hostLevelParams']['java_home']
java_bin = '{java_home}/bin/java'.format(java_home=java_home)
cs_lib_path = self.credential_shell_lib_path
serviceName = commandJson['serviceName']
# Gather the password values and remove them from the configuration
provider_paths = [] # A service may depend on multiple configs
configtype_credentials = self.getConfigTypeCredentials(commandJson)
for config_type, credentials in configtype_credentials.items():
config = commandJson['configurations'][config_type]
file_path = os.path.join(self.getProviderDirectory(serviceName),
"{0}.jceks".format(config_type))
if os.path.exists(file_path):
os.remove(file_path)
provider_path = 'jceks://file{file_path}'.format(
file_path=file_path)
provider_paths.append(provider_path)
logger.info('provider_path={0}'.format(provider_path))
for alias, pwd in credentials.items():
logger.debug("config={0}".format(config))
protected_pwd = PasswordString(pwd)
# Generate the JCEKS file
cmd = (java_bin, '-cp', cs_lib_path, self.credential_shell_cmd,
'create', alias, '-value', protected_pwd, '-provider',
provider_path)
logger.info(cmd)
cmd_result = subprocess.call(cmd)
logger.info('cmd_result = {0}'.format(cmd_result))
os.chmod(
file_path, 0644
) # group and others should have read access so that the service user can read
if provider_paths:
# Add JCEKS provider paths instead
config[self.CREDENTIAL_PROVIDER_PROPERTY_NAME] = ','.join(
provider_paths)
return cmd_result
def setup_ranger_plugin_keystore(service_name, audit_db_is_enabled,
hdp_version, credential_file,
xa_audit_db_password, ssl_truststore_password,
ssl_keystore_password, component_user,
component_group, java_home):
cred_lib_path = format(
'/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install/lib/*')
cred_setup_prefix = (format(
'/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/ranger_credential_helper.py'
), '-l', cred_lib_path)
if audit_db_is_enabled:
cred_setup = cred_setup_prefix + (
'-f', credential_file, '-k', 'auditDBCred', '-v',
PasswordString(xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': java_home},
logoutput=True,
sudo=True)
File(credential_file,
owner=component_user,
group=component_group,
mode=0640)
def ranger_credential_helper(lib_path, alias_key, alias_value, file_path):
import params
java_bin = format('{java_home}/bin/java')
file_path = format('jceks://file{file_path}')
cmd = (java_bin, '-cp', lib_path,
'org.apache.ranger.credentialapi.buildks', 'create', alias_key,
'-value', PasswordString(alias_value), '-provider', file_path)
Execute(cmd,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
def encrypt_sensitive_properties(self, config_version_file, current_version, nifi_config_dir, jdk64_home, nifi_user,
nifi_group, master_key_password, nifi_flow_config_dir, nifi_sensitive_props_key,
is_starting):
Logger.info("Encrypting NiFi sensitive configuration properties")
encrypt_config_script = nifi_toolkit_util.get_toolkit_script('encrypt-config.sh')
encrypt_config_script_prefix = ('JAVA_HOME=' + jdk64_home, encrypt_config_script)
File(encrypt_config_script, mode=0755)
if is_starting:
last_master_key_password = None
last_config_version = nifi_toolkit_util.get_config_version(config_version_file, 'encrypt')
encrypt_config_script_params = ('-v', '-b', nifi_config_dir + '/bootstrap.conf')
encrypt_config_script_params = encrypt_config_script_params + ('-n', nifi_config_dir + '/nifi.properties')
if sudo.path_isfile(nifi_flow_config_dir + '/flow.xml.gz') and len(
sudo.read_file(nifi_flow_config_dir + '/flow.xml.gz')) > 0:
encrypt_config_script_params = encrypt_config_script_params + (
'-f', nifi_flow_config_dir + '/flow.xml.gz', '-s', PasswordString(nifi_sensitive_props_key))
if nifi_toolkit_util.contains_providers(nifi_config_dir + '/login-identity-providers.xml'):
encrypt_config_script_params = encrypt_config_script_params + (
'-l', nifi_config_dir + '/login-identity-providers.xml')
if last_config_version:
last_config = nifi_toolkit_util.get_config_by_version('/var/lib/ambari-agent/data',
'nifi-ambari-config', last_config_version)
last_master_key_password = last_config['configurations']['nifi-ambari-config'][
'nifi.security.encrypt.configuration.password']
if last_master_key_password and last_master_key_password != master_key_password:
encrypt_config_script_params = encrypt_config_script_params + (
'-m', '-w', PasswordString(last_master_key_password))
encrypt_config_script_params = encrypt_config_script_params + ('-p', PasswordString(master_key_password))
encrypt_config_script_prefix = encrypt_config_script_prefix + encrypt_config_script_params
Execute(encrypt_config_script_prefix, user=nifi_user, logoutput=False)
nifi_toolkit_util.save_config_version(config_version_file, 'encrypt', current_version, nifi_user,
nifi_group)
def execute_bootstrap_storage_env(self, params):
from urlparse import urlparse
try:
streamline_storage_database_hostname = \
urlparse(urlparse(params.streamline_storage_connector_connectorURI)[2])[1].split(":")[0]
database_admin_jdbc_url = params.database_admin_jdbc_url
if params.streamline_storage_type == 'postgresql':
database_admin_jdbc_url = database_admin_jdbc_url + '/postgres'
bootstrap_storage_initevn_db_cmd = database_admin_jdbc_url + ' ' + params.database_admin_user_name + ' ' + PasswordString(
params.database_admin_password
) + ' ' + params.streamline_storage_connector_user + ' ' + PasswordString(
params.streamline_storage_connector_password
) + ' ' + params.streamline_storage_database
Execute(params.bootstrap_storage_initevn_run_cmd + ' ' +
bootstrap_storage_initevn_db_cmd,
user='******')
except:
show_logs(params.streamline_log_dir, params.streamline_user)
def do_keystore_setup(cred_provider_path, credential_alias,
credential_password):
import params
if cred_provider_path is not None:
cred_setup = params.cred_setup_prefix + (
'-f', cred_provider_path, '-k', credential_alias, '-v',
PasswordString(credential_password), '-c', '1')
Execute(
cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True,
)
File(cred_provider_path,
owner=params.kms_user,
group=params.kms_group,
mode=0640)
def execute_bootstrap_storage_env(self, params):
from urlparse import urlparse
try:
#Getting hostname where registry's database would be installed.
registry_storage_database_hostname = urlparse(
urlparse(params.registry_storage_connector_connectorURI)
[2])[1].split(":")[0]
database_admin_jdbc_url = params.database_admin_jdbc_url
if params.registry_storage_type == 'postgresql':
database_admin_jdbc_url = database_admin_jdbc_url + '/postgres'
bootstrap_storage_initevn_db_cmd = database_admin_jdbc_url + ' ' + params.database_admin_user_name + ' ' + PasswordString(
params.database_admin_password
) + ' ' + params.registry_storage_connector_user + ' ' + PasswordString(
params.registry_storage_connector_password
) + ' ' + params.registry_storage_database
Execute(params.bootstrap_storage_initevn_run_cmd + ' ' +
bootstrap_storage_initevn_db_cmd,
user='******')
except:
show_logs(params.registry_log_dir, params.registry_user)
raise
oracle_driver_jar_name = "ojdbc6.jar"
oozie_metastore_user_name = config['configurations']['oozie-site'][
'oozie.service.JPAService.jdbc.username']
if credential_store_enabled:
if 'hadoop.security.credential.provider.path' in config['configurations'][
'oozie-site']:
cs_lib_path = config['configurations']['oozie-site'][
'credentialStoreClassPath']
java_home = config['hostLevelParams']['java_home']
alias = 'oozie.service.JPAService.jdbc.password'
provider_path = config['configurations']['oozie-site'][
'hadoop.security.credential.provider.path']
oozie_metastore_user_passwd = PasswordString(
get_password_from_credential_store(alias, provider_path,
cs_lib_path, java_home,
jdk_location))
else:
raise Exception(
"hadoop.security.credential.provider.path property should be set")
else:
oozie_metastore_user_passwd = default(
"/configurations/oozie-site/oozie.service.JPAService.jdbc.password",
"")
oozie_jdbc_connection_url = default(
"/configurations/oozie-site/oozie.service.JPAService.jdbc.url", "")
oozie_log_dir = config['configurations']['oozie-env']['oozie_log_dir']
oozie_data_dir = config['configurations']['oozie-env']['oozie_data_dir']
oozie_server_port = get_port_from_url(
config['configurations']['oozie-site']['oozie.base.url'])
def enable_kms_plugin():
import params
if params.has_ranger_admin:
ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url)
response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(
params.policymgr_mgr_url + '/login.jsp', 'test:test')
if response_code is not None and response_code == 200:
ambari_ranger_admin, ambari_ranger_password = ranger_adm_obj.create_ambari_admin_user(
params.ambari_ranger_admin, params.ambari_ranger_password,
params.admin_uname_password)
ambari_username_password_for_ranger = ambari_ranger_admin + ':' + ambari_ranger_password
else:
raise Fail('Ranger service is not started on given host')
if ambari_ranger_admin != '' and ambari_ranger_password != '':
get_repo_flag = get_repo(params.policymgr_mgr_url,
params.repo_name,
ambari_username_password_for_ranger)
if not get_repo_flag:
create_repo(params.policymgr_mgr_url,
json.dumps(params.kms_ranger_plugin_repo),
ambari_username_password_for_ranger)
else:
raise Fail('Ambari admin username and password not available')
current_datetime = datetime.now()
File(
format('{kms_conf_dir}/ranger-security.xml'),
owner=params.kms_user,
group=params.kms_group,
mode=0644,
content=InlineTemplate(
format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
)))
Directory([
os.path.join('/etc', 'ranger', params.repo_name),
os.path.join('/etc', 'ranger', params.repo_name, 'policycache')
],
owner=params.kms_user,
group=params.kms_group,
mode=0775,
recursive=True)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',
format('kms_{repo_name}.json')),
owner=params.kms_user,
group=params.kms_group,
mode=0644)
XmlConfig(
"ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-audit'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-security'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-policymgr-ssl'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'auditDBCred', '-v',
PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.credential_file,
owner=params.kms_user,
group=params.kms_group,
mode=0640)
def setup_ranger_plugin_keystore(service_name,
audit_db_is_enabled,
credential_file,
xa_audit_db_password,
ssl_truststore_password,
ssl_keystore_password,
component_user,
component_group,
java_home,
cred_lib_path_override=None,
cred_setup_prefix_override=None):
stack_root = Script.get_stack_root().replace('/usr/hdp', '/opt')
service_name = str(service_name).lower()
if cred_lib_path_override is not None:
cred_lib_path = cred_lib_path_override
else:
cred_lib_path = format(
'{stack_root}/ranger-{service_name}-plugin/install/lib/*')
if cred_setup_prefix_override is not None:
cred_setup_prefix = cred_setup_prefix_override
else:
cred_setup_prefix = (format(
'{stack_root}/ranger-{service_name}-plugin/ranger_credential_helper.py'
), '-l', cred_lib_path)
if audit_db_is_enabled:
cred_setup = cred_setup_prefix + (
'-f', credential_file, '-k', 'auditDBCred', '-v',
PasswordString(xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': java_home},
logoutput=True,
sudo=True)
File(credential_file,
owner=component_user,
group=component_group,
mode=0640)
dot_jceks_crc_file_path = os.path.join(
os.path.dirname(credential_file),
"." + os.path.basename(credential_file) + ".crc")
File(dot_jceks_crc_file_path,
owner=component_user,
group=component_group,
only_if=format("test -e {dot_jceks_crc_file_path}"),
mode=0640)
def enable_kms_plugin():
import params
if params.has_ranger_admin:
ranger_flag = False
if params.stack_supports_ranger_kerberos and params.security_enabled:
if not is_empty(params.rangerkms_principal
) and params.rangerkms_principal != '':
ranger_flag = check_ranger_service_support_kerberos(
params.kms_user, params.rangerkms_keytab,
params.rangerkms_principal)
else:
ranger_flag = check_ranger_service_support_kerberos(
params.kms_user, params.spengo_keytab,
params.spnego_principal)
else:
ranger_flag = check_ranger_service()
if not ranger_flag:
Logger.error('Error in Get/Create service for Ranger Kms.')
current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
File(format('{kms_conf_dir}/ranger-security.xml'),
owner=params.kms_user,
group=params.kms_group,
mode=0644,
content=format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'))
Directory([
os.path.join('/etc', 'ranger', params.repo_name),
os.path.join('/etc', 'ranger', params.repo_name, 'policycache')
],
owner=params.kms_user,
group=params.kms_group,
mode=0775,
create_parents=True)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',
format('kms_{repo_name}.json')),
owner=params.kms_user,
group=params.kms_group,
mode=0644)
# remove plain-text password from xml configs
plugin_audit_properties_copy = {}
plugin_audit_properties_copy.update(
params.config['configurations']['ranger-kms-audit'])
if params.plugin_audit_password_property in plugin_audit_properties_copy:
plugin_audit_properties_copy[
params.plugin_audit_password_property] = "crypted"
XmlConfig(
"ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=plugin_audit_properties_copy,
configuration_attributes=params.config['configurationAttributes']
['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-security'],
configuration_attributes=params.config['configurationAttributes']
['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
# remove plain-text password from xml configs
ranger_kms_policymgr_ssl_copy = {}
ranger_kms_policymgr_ssl_copy.update(
params.config['configurations']['ranger-kms-policymgr-ssl'])
for prop in params.kms_plugin_password_properties:
if prop in ranger_kms_policymgr_ssl_copy:
ranger_kms_policymgr_ssl_copy[prop] = "crypted"
XmlConfig(
"ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=ranger_kms_policymgr_ssl_copy,
configuration_attributes=params.config['configurationAttributes']
['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'auditDBCred', '-v',
PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.credential_file,
owner=params.kms_user,
group=params.kms_group,
only_if=format("test -e {credential_file}"),
mode=0640)
dot_jceks_crc_file_path = os.path.join(
os.path.dirname(params.credential_file),
"." + os.path.basename(params.credential_file) + ".crc")
File(dot_jceks_crc_file_path,
owner=params.kms_user,
group=params.kms_group,
only_if=format("test -e {dot_jceks_crc_file_path}"),
mode=0640)
# create ranger kms audit directory
if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node:
try:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/kms",
type="directory",
action="create_on_execute",
owner=params.kms_user,
group=params.kms_group,
mode=0750,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
except Exception, err:
Logger.exception(
"Audit directory creation in HDFS for RANGER KMS Ranger plugin failed with error:\n{0}"
.format(err))
if params.xa_audit_hdfs_is_enabled and len(params.namenode_host) > 1:
Logger.info(
'Audit to Hdfs enabled in NameNode HA environment, creating hdfs-site.xml'
)
XmlConfig(
"hdfs-site.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.
config['configurationAttributes']['hdfs-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644)
else:
File(format('{kms_conf_dir}/hdfs-site.xml'), action="delete")
def enable_kms_plugin():
import params
if params.has_ranger_admin:
ranger_flag = False
if params.stack_supports_ranger_kerberos and params.security_enabled:
if not is_empty(params.rangerkms_principal
) and params.rangerkms_principal != '':
ranger_flag = check_ranger_service_support_kerberos(
params.kms_user, params.rangerkms_keytab,
params.rangerkms_principal)
else:
ranger_flag = check_ranger_service_support_kerberos(
params.kms_user, params.spengo_keytab,
params.spnego_principal)
else:
ranger_flag = check_ranger_service()
if not ranger_flag:
Logger.error('Error in Get/Create service for Ranger Kms.')
current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
File(format('{kms_conf_dir}/ranger-security.xml'),
owner=params.kms_user,
group=params.kms_group,
mode=0644,
content=format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'))
Directory([
os.path.join('/etc', 'ranger', params.repo_name),
os.path.join('/etc', 'ranger', params.repo_name, 'policycache')
],
owner=params.kms_user,
group=params.kms_group,
mode=0775,
create_parents=True)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',
format('kms_{repo_name}.json')),
owner=params.kms_user,
group=params.kms_group,
mode=0644)
XmlConfig(
"ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-audit'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-security'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-policymgr-ssl'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'auditDBCred', '-v',
PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.credential_file,
owner=params.kms_user,
group=params.kms_group,
mode=0640)
cmd = (java_bin, '-cp', cs_lib_path, credential_util_cmd, 'get', alias,
'-provider', provider_path)
cmd_result, std_out_msg = checked_call(cmd)
if cmd_result != 0:
message = 'The following error occurred while executing {0}: {1}'.format(
' '.join(cmd), std_out_msg)
Logger.error(message)
raise
std_out_lines = std_out_msg.split('\n')
passwd = std_out_lines[
-1] # Get the last line of the output, to skip warnings if any.
return passwd
if credential_store_enabled:
hive_metastore_user_passwd = PasswordString(getHiveMetastorePassword())
else:
hive_metastore_user_passwd = config['configurations']['hive-site'][
'javax.jdo.option.ConnectionPassword']
hive_metastore_user_passwd = unicode(
hive_metastore_user_passwd
) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd
hive_metastore_db_type = config['configurations']['hive-env'][
'hive_database_type']
#HACK Temporarily use dbType=azuredb while invoking schematool
if hive_metastore_db_type == "mssql":
hive_metastore_db_type = "azuredb"
#users
hive_user = config['configurations']['hive-env']['hive_user']
def setup_usersync(upgrade_type=None):
import params
usersync_home = params.usersync_home
ranger_home = params.ranger_home
ranger_ugsync_conf = params.ranger_ugsync_conf
if not is_empty(
params.ranger_usersync_ldap_ldapbindpassword
) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder':
password_validation(params.ranger_usersync_ldap_ldapbindpassword)
Directory(params.ranger_pid_dir,
mode=0750,
owner=params.unix_user,
group=params.unix_group)
Directory(params.usersync_log_dir,
owner=params.unix_user,
group=params.unix_group)
Directory(ranger_ugsync_conf,
owner=params.unix_user,
group=params.unix_group,
recursive=True)
if upgrade_type is not None:
src_file = format(
'{usersync_home}/conf.dist/ranger-ugsync-default.xml')
dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
# Added by RAI
src_file = format('{usersync_home}/conf.dist/log4j.xml')
dst_file = format('{usersync_conf}/log4j.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
src_file = format('{usersync_home}/conf.dist/log4j.properties')
dst_file = format('{usersync_conf}/log4j.properties')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
# Added by RAI
# Copies /etc/hadoop/conf.empty/core-site.xml to /etc/ranger/admin/conf/core-site.xml
src_file = format('/etc/hadoop/conf.empty/core-site.xml')
dst_file = format('/etc/ranger/admin/conf/core-site.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
# Added by RAI
#makes 1) ln - sf / etc / ranger / admin / conf / usr / lib / ranger - admin / ews / webapp / WEB - INF / classes
# 2) ln - sf / var / log / ranger // usr / lib / ranger - usersync / logs#
Execute(('ln', '-sf', format('{ranger_conf}'),
format('{ranger_home}/ews/webapp/WEB-INF/classes')),
not_if=format("ls {ranger_home}/ews/webapp/WEB-INF/classes"),
only_if=format("ls {ranger_conf}"),
sudo=True)
Execute(('ln', '-sf', format('{usersync_log_dir}'),
format('{usersync_home}/logs')),
not_if=format("ls {usersync_home}/logs"),
only_if=format("ls {usersync_log_dir}"),
sudo=True)
XmlConfig(
"ranger-ugsync-site.xml",
conf_dir=ranger_ugsync_conf,
configurations=params.config['configurations']['ranger-ugsync-site'],
configuration_attributes=params.config['configuration_attributes']
['ranger-ugsync-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
if os.path.isfile(params.ranger_ugsync_default_file):
File(params.ranger_ugsync_default_file,
owner=params.unix_user,
group=params.unix_group)
if os.path.isfile(params.usgsync_log4j_file):
File(params.usgsync_log4j_file,
owner=params.unix_user,
group=params.unix_group)
if os.path.isfile(params.cred_validator_file):
File(params.cred_validator_file, group=params.unix_group, mode=04555)
cred_file = format('{ranger_home}/ranger_credential_helper.py')
if os.path.isfile(format('{usersync_home}/ranger_credential_helper.py')):
cred_file = format('{usersync_home}/ranger_credential_helper.py')
cred_lib = os.path.join(usersync_home, "lib", "*")
cred_setup_prefix = (cred_file, '-l', cred_lib)
cred_setup = cred_setup_prefix + (
'-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.key.password',
'-v', PasswordString(
params.ranger_usersync_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', params.ugsync_jceks_path, '-k', 'ranger.usersync.ldap.bindalias',
'-v', PasswordString(
params.ranger_usersync_ldap_ldapbindpassword), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', params.ugsync_jceks_path, '-k',
'usersync.ssl.truststore.password', '-v',
PasswordString(params.ranger_usersync_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.ugsync_jceks_path,
owner=params.unix_user,
group=params.unix_group,
mode=0640)
File([params.usersync_start, params.usersync_stop],
owner=params.unix_user,
group=params.unix_group)
File(
params.usersync_services_file,
mode=0755,
)
Execute(('ln', '-sf', format('{usersync_services_file}'),
'/usr/bin/ranger-usersync'),
not_if=format("ls /usr/bin/ranger-usersync"),
only_if=format("ls {usersync_services_file}"),
sudo=True)
if not os.path.isfile(params.ranger_usersync_keystore_file):
cmd = format(
"{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'"
)
Execute(cmd, logoutput=True, user=params.unix_user)
File(params.ranger_usersync_keystore_file,
owner=params.unix_user,
group=params.unix_group,
mode=0640)
'configs': atlas_repository_configuration,
'description': 'atlas repo',
'name': repo_name,
'type': 'atlas',
}
# ranger atlas plugin section end
# atlas admin login username password
atlas_admin_username = config['configurations']['atlas-env'][
'atlas.admin.username']
atlas_admin_password = config['configurations']['atlas-env'][
'atlas.admin.password']
# Atlas Passwords Extracted From Credential Store
if credential_provider:
default_credential_shell_lib_path = jdk_location
truststore_password = PasswordString(
get_password_from_credential_store(
'truststore.password', credential_provider,
os.path.join(default_credential_shell_lib_path, '*'), java64_home,
jdk_location))
keystore_password = PasswordString(
get_password_from_credential_store(
'keystore.password', credential_provider,
os.path.join(default_credential_shell_lib_path, '*'), java64_home,
jdk_location))
key_password = PasswordString(
get_password_from_credential_store(
'password', credential_provider,
os.path.join(default_credential_shell_lib_path, '*'), java64_home,
jdk_location))
def setup_usersync():
import params
Directory(params.ranger_pid_dir,
mode=0750,
owner = params.unix_user,
group = params.unix_group
)
Directory(params.usersync_log_dir,
owner = params.unix_user,
group = params.unix_group
)
Directory(format("{ranger_ugsync_conf}/"),
owner = params.unix_user
)
XmlConfig("ranger-ugsync-site.xml",
conf_dir=params.ranger_ugsync_conf,
configurations=params.config['configurations']['ranger-ugsync-site'],
configuration_attributes=params.config['configuration_attributes']['ranger-ugsync-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
cred_lib = os.path.join(params.usersync_home,"lib","*")
cred_setup_prefix = (format('{ranger_home}/ranger_credential_helper.py'), '-l', cred_lib)
cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.key.password', '-v', PasswordString(params.ranger_usersync_keystore_password), '-c', '1')
Execute(cred_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'ranger.usersync.ldap.bindalias', '-v', PasswordString(params.ranger_usersync_ldap_ldapbindpassword), '-c', '1')
Execute(cred_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.truststore.password', '-v', PasswordString(params.ranger_usersync_truststore_password), '-c', '1')
Execute(cred_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
File(params.ugsync_jceks_path,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
File([params.usersync_start, params.usersync_stop],
owner = params.unix_user,
group = params.unix_group
)
File(params.usersync_services_file,
mode = 0755,
)
Execute(('ln','-sf', format('{usersync_services_file}'),'/usr/bin/ranger-usersync'),
not_if=format("ls /usr/bin/ranger-usersync"),
only_if=format("ls {usersync_services_file}"),
sudo=True)
if not os.path.isfile(params.ranger_usersync_keystore_file):
cmd = format("{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'")
Execute(cmd, logoutput=True, user = params.unix_user)
File(params.ranger_usersync_keystore_file,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
def log_str(self, key, value):
# Hide the passwords from text representations
return repr(PasswordString(value))
def encrypt_sensitive_properties(
nifi_config_dir,
jdk64_home,
java_options,
nifi_user,
last_master_key,
master_key_password,
is_starting,
toolkit_tmp_dir,
stack_version_buildnum,
service,
nifi_flow_config_dir=None,
nifi_sensitive_props_key=None,
support_encrypt_authorizers=None,
):
encrypt_config_script = get_toolkit_script('encrypt-config.sh',
toolkit_tmp_dir,
stack_version_buildnum)
encrypt_config_command = (encrypt_config_script, )
environment = {'JAVA_HOME': jdk64_home, 'JAVA_OPTS': java_options}
File(encrypt_config_script, mode=0755)
if is_starting:
if service == NIFI:
Logger.info("Encrypting NiFi sensitive configuration properties")
encrypt_config_command += ('-v', '-b',
nifi_config_dir + '/bootstrap.conf')
encrypt_config_command += ('-n',
nifi_config_dir + '/nifi.properties')
if (sudo.path_isfile(nifi_flow_config_dir + '/flow.xml.gz') and
len(sudo.read_file(nifi_flow_config_dir + '/flow.xml.gz'))
> 0):
encrypt_config_command += (
'-f', nifi_flow_config_dir + '/flow.xml.gz', '-s',
PasswordString(nifi_sensitive_props_key))
if contains_providers(
nifi_config_dir + '/login-identity-providers.xml',
"provider"):
encrypt_config_command += ('-l', nifi_config_dir +
'/login-identity-providers.xml')
if support_encrypt_authorizers and contains_providers(
nifi_config_dir + '/authorizers.xml', "authorizer"):
encrypt_config_command += ('-a', nifi_config_dir +
'/authorizers.xml')
if last_master_key:
encrypt_config_command += ('-m', '-e',
PasswordString(last_master_key))
encrypt_config_command += ('-p',
PasswordString(master_key_password))
Execute(encrypt_config_command,
user=nifi_user,
logoutput=False,
environment=environment)
elif service == NIFI_REGISTRY:
Logger.info(
"Encrypting NiFi Registry sensitive configuration properties")
encrypt_config_command += ('--nifiRegistry', '-v', '-b',
nifi_config_dir + '/bootstrap.conf')
encrypt_config_command += ('-r', nifi_config_dir +
'/nifi-registry.properties')
if contains_providers(nifi_config_dir + '/identity-providers.xml',
"provider"):
encrypt_config_command += ('-i', nifi_config_dir +
'/identity-providers.xml')
if contains_providers(nifi_config_dir + '/authorizers.xml',
"authorizer"):
encrypt_config_command += ('-a', nifi_config_dir +
'/authorizers.xml')
if last_master_key:
encrypt_config_command += ('--oldKey',
PasswordString(last_master_key))
encrypt_config_command += ('-p',
PasswordString(master_key_password))
Execute(encrypt_config_command,
user=nifi_user,
logoutput=False,
environment=environment)
def enable_kms_plugin():
import params
if params.has_ranger_admin:
count = 0
while count < 5:
ranger_flag = check_ranger_service()
if ranger_flag:
break
else:
time.sleep(5) # delay for 5 seconds
count = count + 1
else:
Logger.error(
"Ranger service is not reachable after {0} tries".format(
count))
current_datetime = datetime.now()
File(
format('{kms_conf_dir}/ranger-security.xml'),
owner=params.kms_user,
group=params.kms_group,
mode=0644,
content=InlineTemplate(
format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
)))
Directory([
os.path.join('/etc', 'ranger', params.repo_name),
os.path.join('/etc', 'ranger', params.repo_name, 'policycache')
],
owner=params.kms_user,
group=params.kms_group,
mode=0775,
create_parents=True)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',
format('kms_{repo_name}.json')),
owner=params.kms_user,
group=params.kms_group,
mode=0644)
XmlConfig(
"ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-audit'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-security'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-policymgr-ssl'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'auditDBCred', '-v',
PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.credential_file,
owner=params.kms_user,
group=params.kms_group,
mode=0640)
def setup_usersync(rolling_upgrade=False):
import params
usersync_home = params.usersync_home
ranger_ugsync_conf = params.ranger_ugsync_conf
if not is_empty(
params.ranger_usersync_ldap_ldapbindpassword
) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder':
password_validation(params.ranger_usersync_ldap_ldapbindpassword)
if rolling_upgrade:
usersync_home = format("/usr/hdp/{version}/ranger-usersync")
ranger_ugsync_conf = format("/usr/hdp/{version}/ranger-usersync/conf")
Directory(params.ranger_pid_dir,
mode=0750,
owner=params.unix_user,
group=params.unix_group)
Directory(params.usersync_log_dir,
owner=params.unix_user,
group=params.unix_group)
Directory(format("{ranger_ugsync_conf}/"), owner=params.unix_user)
if rolling_upgrade:
src_file = format(
'{usersync_home}/conf.dist/ranger-ugsync-default.xml')
dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
src_file = format('{usersync_home}/conf.dist/log4j.xml')
dst_file = format('{usersync_home}/conf/log4j.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
XmlConfig(
"ranger-ugsync-site.xml",
conf_dir=ranger_ugsync_conf,
configurations=params.config['configurations']['ranger-ugsync-site'],
configuration_attributes=params.config['configuration_attributes']
['ranger-ugsync-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
if os.path.isfile(params.ranger_ugsync_default_file):
File(params.ranger_ugsync_default_file,
owner=params.unix_user,
group=params.unix_group)
if os.path.isfile(params.usgsync_log4j_file):
File(params.usgsync_log4j_file,
owner=params.unix_user,
group=params.unix_group)
if os.path.isfile(params.cred_validator_file):
File(params.cred_validator_file, group=params.unix_group, mode=04555)
cred_lib = os.path.join(usersync_home, "lib", "*")
cred_setup_prefix = (format('{usersync_home}/ranger_credential_helper.py'),
'-l', cred_lib)
cred_setup = cred_setup_prefix + (
'-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.key.password',
'-v', PasswordString(
params.ranger_usersync_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', params.ugsync_jceks_path, '-k', 'ranger.usersync.ldap.bindalias',
'-v', PasswordString(
params.ranger_usersync_ldap_ldapbindpassword), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', params.ugsync_jceks_path, '-k',
'usersync.ssl.truststore.password', '-v',
PasswordString(params.ranger_usersync_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.ugsync_jceks_path,
owner=params.unix_user,
group=params.unix_group,
mode=0640)
File([params.usersync_start, params.usersync_stop],
owner=params.unix_user,
group=params.unix_group)
File(
params.usersync_services_file,
mode=0755,
)
Execute(('ln', '-sf', format('{usersync_services_file}'),
'/usr/bin/ranger-usersync'),
not_if=format("ls /usr/bin/ranger-usersync"),
only_if=format("ls {usersync_services_file}"),
sudo=True)
if not os.path.isfile(params.ranger_usersync_keystore_file):
cmd = format(
"{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'"
)
Execute(cmd, logoutput=True, user=params.unix_user)
File(params.ranger_usersync_keystore_file,
owner=params.unix_user,
group=params.unix_group,
mode=0640)