def check_ranger_service():
import params
ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url)
ambari_username_password_for_ranger = format("{ambari_ranger_admin}:{ambari_ranger_password}")
response_code = ranger_adm_obj.check_ranger_login_urllib2(params.policymgr_mgr_url)
if response_code is not None and response_code == 200:
user_resp_code = ranger_adm_obj.create_ambari_admin_user(params.ambari_ranger_admin, params.ambari_ranger_password, params.admin_uname_password)
if user_resp_code is not None and user_resp_code == 200:
get_repo_flag = get_repo(params.policymgr_mgr_url, params.repo_name, ambari_username_password_for_ranger)
if not get_repo_flag:
create_repo_flag = create_repo(params.policymgr_mgr_url, json.dumps(params.kms_ranger_plugin_repo), ambari_username_password_for_ranger)
if create_repo_flag:
return True
else:
return False
else:
return True
else:
Logger.error('Ambari admin user creation failed')
return False
else:
Logger.error('Ranger service is not reachable host')
return False
def check_ranger_service():
import params
policymgr_mgr_url = params.policymgr_mgr_url
if policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url)
ambari_username_password_for_ranger = format(
"{ambari_ranger_admin}:{ambari_ranger_password}")
response_code = ranger_adm_obj.check_ranger_login_urllib2(
policymgr_mgr_url)
if response_code is not None and response_code == 200:
user_resp_code = ranger_adm_obj.create_ambari_admin_user(
params.ambari_ranger_admin, params.ambari_ranger_password,
params.admin_uname_password)
if user_resp_code is not None and user_resp_code == 200:
get_repo_flag = get_repo(policymgr_mgr_url, params.repo_name,
ambari_username_password_for_ranger)
if not get_repo_flag:
return create_repo(policymgr_mgr_url,
json.dumps(params.kms_ranger_plugin_repo),
ambari_username_password_for_ranger)
else:
return True
else:
return False
else:
Logger.error('Ranger service is not reachable')
return False
def setup_ranger_plugin(component_select_name, service_name,
downloaded_custom_connector, driver_curl_source,
driver_curl_target, java_home,
repo_name, plugin_repo_dict,
ranger_env_properties, plugin_properties,
policy_user, policymgr_mgr_url,
plugin_enabled, component_user, component_group, api_version=None, skip_if_rangeradmin_down = True, **kwargs):
File(downloaded_custom_connector,
content = DownloadSource(driver_curl_source),
mode = 0644
)
Execute(('cp', '--remove-destination', downloaded_custom_connector, driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True
)
File(driver_curl_target, mode=0644)
hdp_version = get_hdp_version(component_select_name)
file_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install.properties')
if not os.path.isfile(file_path):
raise Fail(format('Ranger {service_name} plugin install.properties file does not exist at {file_path}'))
ModifyPropertiesFile(file_path,
properties = plugin_properties
)
custom_plugin_properties = dict()
custom_plugin_properties['CUSTOM_USER'] = component_user
custom_plugin_properties['CUSTOM_GROUP'] = component_group
ModifyPropertiesFile(file_path,properties = custom_plugin_properties)
if plugin_enabled:
cmd = (format('enable-{service_name}-plugin.sh'),)
if api_version == 'v2' and api_version is not None:
ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down)
else:
ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
policy_user)
else:
cmd = (format('disable-{service_name}-plugin.sh'),)
cmd_env = {'JAVA_HOME': java_home, 'PWD': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin'), 'PATH': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin')}
Execute(cmd,
environment=cmd_env,
logoutput=True,
sudo=True,
)
def setup_ranger_hdfs():
import params
if params.has_ranger_admin:
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source))
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
params.driver_curl_target),
path=["/bin", "/usr/bin/"],
not_if=format("test -f {driver_curl_target}"),
sudo=True)
hdp_version = get_hdp_version('hadoop-client')
file_path = format(
'/usr/hdp/{hdp_version}/ranger-hdfs-plugin/install.properties')
if not os.path.isfile(file_path):
raise Fail(
format(
'Ranger HBase plugin install.properties file does not exist at {file_path}'
))
ModifyPropertiesFile(file_path,
properties=params.config['configurations']
['ranger-hdfs-plugin-properties'])
if params.enable_ranger_hdfs:
cmd = ('enable-hdfs-plugin.sh', )
ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url)
ranger_adm_obj.create_ranger_repository(
'hdfs', params.repo_name, params.hdfs_ranger_plugin_repo,
params.ambari_ranger_admin, params.ambari_ranger_password,
params.admin_uname, params.admin_password, params.policy_user)
else:
cmd = ('disable-hdfs-plugin.sh', )
cmd_env = {
'JAVA_HOME': params.java_home,
'PWD': format('/usr/hdp/{hdp_version}/ranger-hdfs-plugin'),
'PATH': format('/usr/hdp/{hdp_version}/ranger-hdfs-plugin')
}
Execute(
cmd,
environment=cmd_env,
logoutput=True,
sudo=True,
)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_plugin(component_select_name, service_name,
downloaded_custom_connector, driver_curl_source,
driver_curl_target, java_home,
repo_name, plugin_repo_dict,
ranger_env_properties, plugin_properties,
policy_user, policymgr_mgr_url,
plugin_enabled):
File(downloaded_custom_connector,
content = DownloadSource(driver_curl_source)
)
Execute(('cp', '--remove-destination', downloaded_custom_connector, driver_curl_target),
not_if=format("test -f {driver_curl_target}"),
sudo=True
)
hdp_version = get_hdp_version(component_select_name)
file_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install.properties')
if not os.path.isfile(file_path):
raise Fail(format('Ranger {service_name} plugin install.properties file does not exist at {file_path}'))
ModifyPropertiesFile(file_path,
properties = plugin_properties
)
if plugin_enabled:
cmd = (format('enable-{service_name}-plugin.sh'),)
ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
policy_user)
else:
cmd = (format('disable-{service_name}-plugin.sh'),)
cmd_env = {'JAVA_HOME': java_home, 'PWD': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin'), 'PATH': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin')}
Execute(cmd,
environment=cmd_env,
logoutput=True,
sudo=True,
)
def setup_ranger_hbase():
import params
if params.has_ranger_admin:
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source)
)
if not os.path.isfile(params.driver_curl_target):
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
try:
command = 'hdp-select status hbase-client'
return_code, hdp_output = shell.call(command, timeout=20)
except Exception, e:
Logger.error(str(e))
raise Fail('Unable to execute hdp-select command to retrieve the version.')
if return_code != 0:
raise Fail('Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))
hdp_version = re.sub('hbase-client - ', '', hdp_output).strip()
match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
if match is None:
raise Fail('Failed to get extracted version')
file_path = '/usr/hdp/'+ hdp_version +'/ranger-hbase-plugin/install.properties'
if not os.path.isfile(file_path):
raise Fail('Ranger HBase plugin install.properties file does not exist at {0}'.format(file_path))
ranger_hbase_dict = ranger_hbase_properties()
hbase_repo_data = hbase_repo_properties()
write_properties_to_file(file_path, ranger_hbase_dict)
if params.enable_ranger_hbase:
cmd = format('cd /usr/hdp/{hdp_version}/ranger-hbase-plugin/ && sh enable-hbase-plugin.sh')
ranger_adm_obj = Rangeradmin(url=ranger_hbase_dict['POLICY_MGR_URL'])
response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(ranger_hbase_dict['POLICY_MGR_URL'] + '/login.jsp', 'test:test')
if response_code is not None and response_code == 200:
ambari_ranger_admin, ambari_ranger_password = ranger_adm_obj.create_ambari_admin_user(params.ambari_ranger_admin, params.ambari_ranger_password, params.admin_uname_password)
ambari_username_password_for_ranger = ambari_ranger_admin + ':' + ambari_ranger_password
if ambari_ranger_admin != '' and ambari_ranger_password != '':
repo = ranger_adm_obj.get_repository_by_name_urllib2(ranger_hbase_dict['REPOSITORY_NAME'], 'hbase', 'true', ambari_username_password_for_ranger)
if repo and repo['name'] == ranger_hbase_dict['REPOSITORY_NAME']:
Logger.info('Hbase Repository exist')
else:
response = ranger_adm_obj.create_repository_urllib2(hbase_repo_data, ambari_username_password_for_ranger, params.policy_user)
if response is not None:
Logger.info('Hbase Repository created in Ranger admin')
else:
Logger.info('Hbase Repository creation failed in Ranger admin')
else:
Logger.info('Ambari admin username and password are blank ')
else:
Logger.info('Ranger service is not started on given host')
else:
cmd = format('cd /usr/hdp/{hdp_version}/ranger-hbase-plugin/ && sh disable-hbase-plugin.sh')
Execute(cmd, environment={'JAVA_HOME': params.java64_home}, logoutput=True)
def enable_kms_plugin():
import params
if params.has_ranger_admin:
ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url)
response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(params.policymgr_mgr_url + '/login.jsp', 'test:test')
if response_code is not None and response_code == 200:
ambari_ranger_admin, ambari_ranger_password = ranger_adm_obj.create_ambari_admin_user(params.ambari_ranger_admin, params.ambari_ranger_password, params.admin_uname_password)
ambari_username_password_for_ranger = ambari_ranger_admin + ':' + ambari_ranger_password
else:
raise Fail('Ranger service is not started on given host')
if ambari_ranger_admin != '' and ambari_ranger_password != '':
get_repo_flag = get_repo(params.policymgr_mgr_url, params.repo_name, ambari_username_password_for_ranger)
if not get_repo_flag:
create_repo(params.policymgr_mgr_url, json.dumps(params.kms_ranger_plugin_repo), ambari_username_password_for_ranger)
else:
raise Fail('Ambari admin username and password not available')
current_datetime = datetime.now()
File(format('{kms_conf_dir}/ranger-security.xml'),
owner = params.kms_user,
group = params.kms_group,
mode = 0644,
content = InlineTemplate(format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'))
)
Directory([os.path.join('/etc', 'ranger', params.repo_name), os.path.join('/etc', 'ranger', params.repo_name, 'policycache')],
owner = params.kms_user,
group = params.kms_group,
mode=0775,
recursive = True
)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',format('kms_{repo_name}.json')),
owner = params.kms_user,
group = params.kms_group,
mode = 0644
)
XmlConfig("ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-audit'],
configuration_attributes=params.config['configuration_attributes']['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig("ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-security'],
configuration_attributes=params.config['configuration_attributes']['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-policymgr-ssl'],
configuration_attributes=params.config['configuration_attributes']['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'auditDBCred', '-v', PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslKeyStore', '-v', PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslTrustStore', '-v', PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
File(params.credential_file,
owner = params.kms_user,
group = params.kms_group,
mode = 0640
)
def enable_kms_plugin():
import params
if params.has_ranger_admin:
ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url)
response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(
params.policymgr_mgr_url + '/login.jsp', 'test:test')
if response_code is not None and response_code == 200:
ambari_ranger_admin, ambari_ranger_password = ranger_adm_obj.create_ambari_admin_user(
params.ambari_ranger_admin, params.ambari_ranger_password,
params.admin_uname_password)
ambari_username_password_for_ranger = ambari_ranger_admin + ':' + ambari_ranger_password
else:
raise Fail('Ranger service is not started on given host')
if ambari_ranger_admin != '' and ambari_ranger_password != '':
get_repo_flag = get_repo(params.policymgr_mgr_url,
params.repo_name,
ambari_username_password_for_ranger)
if not get_repo_flag:
create_repo(params.policymgr_mgr_url,
json.dumps(params.kms_ranger_plugin_repo),
ambari_username_password_for_ranger)
else:
raise Fail('Ambari admin username and password not available')
current_datetime = datetime.now()
File(
format('{kms_conf_dir}/ranger-security.xml'),
owner=params.kms_user,
group=params.kms_group,
mode=0644,
content=InlineTemplate(
format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
)))
Directory([
os.path.join('/etc', 'ranger', params.repo_name),
os.path.join('/etc', 'ranger', params.repo_name, 'policycache')
],
owner=params.kms_user,
group=params.kms_group,
mode=0775,
recursive=True)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',
format('kms_{repo_name}.json')),
owner=params.kms_user,
group=params.kms_group,
mode=0644)
XmlConfig(
"ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-audit'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-security'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-policymgr-ssl'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'auditDBCred', '-v',
PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.credential_file,
owner=params.kms_user,
group=params.kms_group,
mode=0640)
def setup_ranger_plugin(component_select_name,
service_name,
previous_jdbc_jar,
component_downloaded_custom_connector,
component_driver_curl_source,
component_driver_curl_target,
java_home,
repo_name,
plugin_repo_dict,
ranger_env_properties,
plugin_properties,
policy_user,
policymgr_mgr_url,
plugin_enabled,
conf_dict,
component_user,
component_group,
cache_service_list,
plugin_audit_properties,
plugin_audit_attributes,
plugin_security_properties,
plugin_security_attributes,
plugin_policymgr_ssl_properties,
plugin_policymgr_ssl_attributes,
component_list,
audit_db_is_enabled,
credential_file,
xa_audit_db_password,
ssl_truststore_password,
ssl_keystore_password,
api_version=None,
stack_version_override=None,
skip_if_rangeradmin_down=True,
is_security_enabled=False,
is_stack_supports_ranger_kerberos=False,
component_user_principal=None,
component_user_keytab=None,
cred_lib_path_override=None,
cred_setup_prefix_override=None):
if audit_db_is_enabled and component_driver_curl_source is not None and not component_driver_curl_source.endswith(
"/None"):
if previous_jdbc_jar and os.path.isfile(previous_jdbc_jar):
File(previous_jdbc_jar, action='delete')
File(component_downloaded_custom_connector,
content=DownloadSource(component_driver_curl_source),
mode=0644)
Execute(('cp', '--remove-destination',
component_downloaded_custom_connector,
component_driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(component_driver_curl_target, mode=0644)
if policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
if stack_version_override is None:
stack_version = get_stack_version(component_select_name)
else:
stack_version = stack_version_override
component_conf_dir = conf_dict
if plugin_enabled:
service_name_exist = False
policycache_path = os.path.join('/etc', 'ranger', repo_name,
'policycache')
try:
for cache_service in cache_service_list:
policycache_json_file = format(
'{policycache_path}/{cache_service}_{repo_name}.json')
if os.path.isfile(policycache_json_file) and os.path.getsize(
policycache_json_file) > 0:
with open(policycache_json_file) as json_file:
json_data = json.load(json_file)
if 'serviceName' in json_data and json_data[
'serviceName'] == repo_name:
service_name_exist = True
Logger.info(
"Skipping Ranger API calls, as policy cache file exists for {0}"
.format(service_name))
Logger.warning(
"If service name for {0} is not created on Ranger Admin UI, then to re-create it delete policy cache file: {1}"
.format(service_name, policycache_json_file))
break
except Exception, err:
Logger.error(
"Error occurred while fetching service name from policy cache file.\nError: {0}"
.format(err))
if not service_name_exist:
if api_version is not None and api_version == 'v2':
ranger_adm_obj = RangeradminV2(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(
service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'],
ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'],
ranger_env_properties['admin_password'], policy_user,
is_security_enabled, is_stack_supports_ranger_kerberos,
component_user, component_user_principal,
component_user_keytab)
else:
ranger_adm_obj = Rangeradmin(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(
service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'],
ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'],
ranger_env_properties['admin_password'], policy_user)
current_datetime = datetime.now()
File(
format('{component_conf_dir}/ranger-security.xml'),
owner=component_user,
group=component_group,
mode=0644,
content=InlineTemplate(
format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
)))
Directory([
os.path.join('/etc', 'ranger', repo_name),
os.path.join('/etc', 'ranger', repo_name, 'policycache')
],
owner=component_user,
group=component_group,
mode=0775,
create_parents=True,
cd_access='a')
for cache_service in cache_service_list:
File(os.path.join('/etc', 'ranger', repo_name, 'policycache',
format('{cache_service}_{repo_name}.json')),
owner=component_user,
group=component_group,
mode=0644)
# remove plain-text password from xml configs
plugin_audit_password_property = 'xasecure.audit.destination.db.password'
plugin_audit_properties_copy = {}
plugin_audit_properties_copy.update(plugin_audit_properties)
if plugin_audit_password_property in plugin_audit_properties_copy:
plugin_audit_properties_copy[
plugin_audit_password_property] = "crypted"
XmlConfig(format('ranger-{service_name}-audit.xml'),
conf_dir=component_conf_dir,
configurations=plugin_audit_properties_copy,
configuration_attributes=plugin_audit_attributes,
owner=component_user,
group=component_group,
mode=0744)
XmlConfig(format('ranger-{service_name}-security.xml'),
conf_dir=component_conf_dir,
configurations=plugin_security_properties,
configuration_attributes=plugin_security_attributes,
owner=component_user,
group=component_group,
mode=0744)
# remove plain-text password from xml configs
plugin_password_properties = [
'xasecure.policymgr.clientssl.keystore.password',
'xasecure.policymgr.clientssl.truststore.password'
]
plugin_policymgr_ssl_properties_copy = {}
plugin_policymgr_ssl_properties_copy.update(
plugin_policymgr_ssl_properties)
for prop in plugin_password_properties:
if prop in plugin_policymgr_ssl_properties_copy:
plugin_policymgr_ssl_properties_copy[prop] = "crypted"
if str(service_name).lower() == 'yarn':
XmlConfig("ranger-policymgr-ssl-yarn.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties_copy,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
else:
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties_copy,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
# creating symblink should be done by rpm package
# setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list)
setup_ranger_plugin_keystore(
service_name, audit_db_is_enabled, stack_version, credential_file,
xa_audit_db_password, ssl_truststore_password,
ssl_keystore_password, component_user, component_group, java_home,
cred_lib_path_override, cred_setup_prefix_override)
def setup_ranger_hive():
import params
if params.has_ranger_admin:
File(
params.ranger_downloaded_custom_connector,
content=DownloadSource(params.ranger_driver_curl_source),
)
if not os.path.isfile(params.ranger_driver_curl_target):
Execute(('cp', '--remove-destination',
params.ranger_downloaded_custom_connector,
params.ranger_driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
try:
command = 'hdp-select status hive-server2'
return_code, hdp_output = shell.call(command, timeout=20)
except Exception, e:
Logger.error(str(e))
raise Fail(
'Unable to execute hdp-select command to retrieve the version.'
)
if return_code != 0:
raise Fail(
'Unable to determine the current version because of a non-zero return code of {0}'
.format(str(return_code)))
hdp_version = re.sub('hive-server2 - ', '', hdp_output).strip()
match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
if match is None:
raise Fail('Failed to get extracted version')
file_path = '/usr/hdp/' + hdp_version + '/ranger-hive-plugin/install.properties'
if not os.path.isfile(file_path):
raise Fail(
'Ranger Hive plugin install.properties file does not exist at {0}'
.format(file_path))
ranger_hive_dict = ranger_hive_properties()
hive_repo_data = hive_repo_properties()
write_properties_to_file(file_path, ranger_hive_dict)
if params.enable_ranger_hive:
cmd = format(
'cd /usr/hdp/{hdp_version}/ranger-hive-plugin/ && sh enable-hive-plugin.sh'
)
ranger_adm_obj = Rangeradmin(
url=ranger_hive_dict['POLICY_MGR_URL'])
response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(
ranger_hive_dict['POLICY_MGR_URL'] + '/login.jsp', 'test:test')
if response_code is not None and response_code == 200:
ambari_ranger_admin, ambari_ranger_password = ranger_adm_obj.create_ambari_admin_user(
params.ambari_ranger_admin, params.ambari_ranger_password,
params.admin_uname_password)
ambari_username_password_for_ranger = ambari_ranger_admin + ':' + ambari_ranger_password
if ambari_ranger_admin != '' and ambari_ranger_password != '':
repo = ranger_adm_obj.get_repository_by_name_urllib2(
ranger_hive_dict['REPOSITORY_NAME'], 'hive', 'true',
ambari_username_password_for_ranger)
if repo and repo['name'] == ranger_hive_dict[
'REPOSITORY_NAME']:
Logger.info('Hive Repository exist')
else:
response = ranger_adm_obj.create_repository_urllib2(
hive_repo_data,
ambari_username_password_for_ranger,
params.policy_user)
if response is not None:
Logger.info(
'Hive Repository created in Ranger Admin')
else:
Logger.info(
'Hive Repository creation failed in Ranger Admin'
)
else:
Logger.info(
'Ambari admin username and password are blank ')
else:
Logger.info('Ranger service is not started on given host')
else:
cmd = format(
'cd /usr/hdp/{hdp_version}/ranger-hive-plugin/ && sh disable-hive-plugin.sh'
)
Execute(cmd,
environment={'JAVA_HOME': params.java64_home},
logoutput=True)
def setup_ranger_plugin(component_select_name,
service_name,
component_downloaded_custom_connector,
component_driver_curl_source,
component_driver_curl_target,
java_home,
repo_name,
plugin_repo_dict,
ranger_env_properties,
plugin_properties,
policy_user,
policymgr_mgr_url,
plugin_enabled,
conf_dict,
component_user,
component_group,
cache_service_list,
plugin_audit_properties,
plugin_audit_attributes,
plugin_security_properties,
plugin_security_attributes,
plugin_policymgr_ssl_properties,
plugin_policymgr_ssl_attributes,
component_list,
audit_db_is_enabled,
credential_file,
xa_audit_db_password,
ssl_truststore_password,
ssl_keystore_password,
api_version=None,
hdp_version_override=None,
skip_if_rangeradmin_down=True):
if audit_db_is_enabled:
File(component_downloaded_custom_connector,
content=DownloadSource(component_driver_curl_source),
mode=0644)
Execute(('cp', '--remove-destination',
component_downloaded_custom_connector,
component_driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(component_driver_curl_target, mode=0644)
hdp_version = get_hdp_version(component_select_name)
if hdp_version_override is not None:
hdp_version = hdp_version_override
component_conf_dir = conf_dict
if plugin_enabled:
if api_version == 'v2' and api_version is not None:
ranger_adm_obj = RangeradminV2(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
else:
ranger_adm_obj = Rangeradmin(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(
service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'],
ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'],
ranger_env_properties['admin_password'], policy_user)
current_datetime = datetime.now()
File(
format('{component_conf_dir}/ranger-security.xml'),
owner=component_user,
group=component_group,
mode=0644,
content=InlineTemplate(
format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
)))
Directory([
os.path.join('/etc', 'ranger', repo_name),
os.path.join('/etc', 'ranger', repo_name, 'policycache')
],
owner=component_user,
group=component_group,
mode=0775,
create_parents=True,
cd_access='a')
for cache_service in cache_service_list:
File(os.path.join('/etc', 'ranger', repo_name, 'policycache',
format('{cache_service}_{repo_name}.json')),
owner=component_user,
group=component_group,
mode=0644)
XmlConfig(format('ranger-{service_name}-audit.xml'),
conf_dir=component_conf_dir,
configurations=plugin_audit_properties,
configuration_attributes=plugin_audit_attributes,
owner=component_user,
group=component_group,
mode=0744)
XmlConfig(format('ranger-{service_name}-security.xml'),
conf_dir=component_conf_dir,
configurations=plugin_security_properties,
configuration_attributes=plugin_security_attributes,
owner=component_user,
group=component_group,
mode=0744)
if str(service_name).lower() == 'yarn':
XmlConfig("ranger-policymgr-ssl-yarn.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
else:
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
#This should be done by rpm
#setup_ranger_plugin_jar_symblink(hdp_version, service_name, component_list)
setup_ranger_plugin_keystore(service_name, audit_db_is_enabled,
hdp_version, credential_file,
xa_audit_db_password,
ssl_truststore_password,
ssl_keystore_password, component_user,
component_group, java_home)
else:
File(format('{component_conf_dir}/ranger-security.xml'),
action="delete")
def setup_ranger_plugin(component_select_name,
service_name,
previous_jdbc_jar,
component_downloaded_custom_connector,
component_driver_curl_source,
component_driver_curl_target,
java_home,
repo_name,
plugin_repo_dict,
ranger_env_properties,
plugin_properties,
policy_user,
policymgr_mgr_url,
plugin_enabled,
conf_dict,
component_user,
component_group,
cache_service_list,
plugin_audit_properties,
plugin_audit_attributes,
plugin_security_properties,
plugin_security_attributes,
plugin_policymgr_ssl_properties,
plugin_policymgr_ssl_attributes,
component_list,
audit_db_is_enabled,
credential_file,
xa_audit_db_password,
ssl_truststore_password,
ssl_keystore_password,
api_version=None,
stack_version_override=None,
skip_if_rangeradmin_down=True,
is_security_enabled=False,
is_stack_supports_ranger_kerberos=False,
component_user_principal=None,
component_user_keytab=None,
cred_lib_path_override=None,
cred_setup_prefix_override=None):
if audit_db_is_enabled and component_driver_curl_source is not None and not component_driver_curl_source.endswith(
"/None"):
if previous_jdbc_jar and os.path.isfile(previous_jdbc_jar):
File(previous_jdbc_jar, action='delete')
File(component_downloaded_custom_connector,
content=DownloadSource(component_driver_curl_source),
mode=0644)
Execute(('cp', '--remove-destination',
component_downloaded_custom_connector,
component_driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(component_driver_curl_target, mode=0644)
if policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
if stack_version_override is None:
stack_version = get_stack_version(component_select_name)
else:
stack_version = stack_version_override
component_conf_dir = conf_dict
if plugin_enabled:
service_name_exist = get_policycache_service_name(
service_name, repo_name, cache_service_list)
if not service_name_exist:
if api_version is not None and api_version == 'v2':
ranger_adm_obj = RangeradminV2(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(
service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'],
ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'],
ranger_env_properties['admin_password'], policy_user,
is_security_enabled, is_stack_supports_ranger_kerberos,
component_user, component_user_principal,
component_user_keytab)
else:
ranger_adm_obj = Rangeradmin(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(
service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'],
ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'],
ranger_env_properties['admin_password'], policy_user)
current_datetime = datetime.now()
File(
format('{component_conf_dir}/ranger-security.xml'),
owner=component_user,
group=component_group,
mode=0644,
content=InlineTemplate(
format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
)))
Directory([
os.path.join('/etc', 'ranger', repo_name),
os.path.join('/etc', 'ranger', repo_name, 'policycache')
],
owner=component_user,
group=component_group,
mode=0775,
create_parents=True,
cd_access='a')
for cache_service in cache_service_list:
File(os.path.join('/etc', 'ranger', repo_name, 'policycache',
format('{cache_service}_{repo_name}.json')),
owner=component_user,
group=component_group,
mode=0644)
# remove plain-text password from xml configs
plugin_audit_password_property = 'xasecure.audit.destination.db.password'
plugin_audit_properties_copy = {}
plugin_audit_properties_copy.update(plugin_audit_properties)
if plugin_audit_password_property in plugin_audit_properties_copy:
plugin_audit_properties_copy[
plugin_audit_password_property] = "crypted"
XmlConfig(format('ranger-{service_name}-audit.xml'),
conf_dir=component_conf_dir,
configurations=plugin_audit_properties_copy,
configuration_attributes=plugin_audit_attributes,
owner=component_user,
group=component_group,
mode=0744)
XmlConfig(format('ranger-{service_name}-security.xml'),
conf_dir=component_conf_dir,
configurations=plugin_security_properties,
configuration_attributes=plugin_security_attributes,
owner=component_user,
group=component_group,
mode=0744)
# remove plain-text password from xml configs
plugin_password_properties = [
'xasecure.policymgr.clientssl.keystore.password',
'xasecure.policymgr.clientssl.truststore.password'
]
plugin_policymgr_ssl_properties_copy = {}
plugin_policymgr_ssl_properties_copy.update(
plugin_policymgr_ssl_properties)
for prop in plugin_password_properties:
if prop in plugin_policymgr_ssl_properties_copy:
plugin_policymgr_ssl_properties_copy[prop] = "crypted"
if str(service_name).lower() == 'yarn':
XmlConfig("ranger-policymgr-ssl-yarn.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties_copy,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
else:
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties_copy,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
setup_ranger_plugin_keystore(
service_name, audit_db_is_enabled, stack_version, credential_file,
xa_audit_db_password, ssl_truststore_password,
ssl_keystore_password, component_user, component_group, java_home,
cred_lib_path_override, cred_setup_prefix_override)
else:
File(format('{component_conf_dir}/ranger-security.xml'),
action="delete")
def kms():
import params
if params.has_ranger_admin:
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source)
)
Directory(params.java_share_dir,
mode=0755
)
if not os.path.isfile(params.driver_curl_target):
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
XmlConfig("kms-acls.xml",
conf_dir=params.kms_config_dir,
configurations=params.config['configurations']['kms-acls'],
configuration_attributes=params.config['configuration_attributes']['kms-acls'],
owner=params.kms_user,
group=params.kms_group
)
XmlConfig("kms-site.xml",
conf_dir=params.kms_config_dir,
configurations=params.config['configurations']['kms-site'],
configuration_attributes=params.config['configuration_attributes']['kms-site'],
owner=params.kms_user,
group=params.kms_group
)
File(os.path.join(params.kms_config_dir, "kms-log4j.properties"),
owner=params.kms_user,
group=params.kms_group,
content=params.kms_log4j
)
repo_data = kms_repo_properties()
ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url)
response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(params.policymgr_mgr_url + '/login.jsp', 'test:test')
if response_code is not None and response_code == 200:
ambari_ranger_admin, ambari_ranger_password = ranger_adm_obj.create_ambari_admin_user(params.ambari_ranger_admin, params.ambari_ranger_password, params.admin_uname_password)
ambari_username_password_for_ranger = ambari_ranger_admin + ':' + ambari_ranger_password
else:
raise Fail('Ranger service is not started on given host')
if ambari_ranger_admin != '' and ambari_ranger_password != '':
get_repo_flag = get_repo(params.policymgr_mgr_url, params.repo_name, ambari_username_password_for_ranger)
if not get_repo_flag:
create_repo(params.policymgr_mgr_url, repo_data, ambari_username_password_for_ranger)
else:
raise Fail('Ambari admin username and password not available')
file_path = format('{kms_home}/install.properties')
ranger_kms_dict = ranger_kms_properties()
write_properties_to_file(file_path, ranger_kms_dict)
env_dict = {'JAVA_HOME': params.java_home, 'RANGER_HOME': params.kms_home}
setup_sh = format("cd {kms_home} && ") + as_sudo([format('{kms_home}/setup.sh')])
Execute(setup_sh, environment=env_dict, logoutput=True)
def setup_ranger_plugin(component_select_name, service_name, previous_jdbc_jar,
component_downloaded_custom_connector, component_driver_curl_source,
component_driver_curl_target, java_home,
repo_name, plugin_repo_dict,
ranger_env_properties, plugin_properties,
policy_user, policymgr_mgr_url,
plugin_enabled, conf_dict, component_user, component_group,
cache_service_list, plugin_audit_properties, plugin_audit_attributes,
plugin_security_properties, plugin_security_attributes,
plugin_policymgr_ssl_properties, plugin_policymgr_ssl_attributes,
component_list, audit_db_is_enabled, credential_file,
xa_audit_db_password, ssl_truststore_password,
ssl_keystore_password, api_version=None, stack_version_override = None, skip_if_rangeradmin_down = True,
is_security_enabled = False, is_stack_supports_ranger_kerberos = False,
component_user_principal = None, component_user_keytab = None):
if audit_db_is_enabled and component_driver_curl_source is not None and not component_driver_curl_source.endswith("/None"):
if previous_jdbc_jar and os.path.isfile(previous_jdbc_jar):
File(previous_jdbc_jar, action='delete')
File(component_downloaded_custom_connector,
content = DownloadSource(component_driver_curl_source),
mode = 0644
)
Execute(('cp', '--remove-destination', component_downloaded_custom_connector, component_driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True
)
File(component_driver_curl_target, mode=0644)
if policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
stack_version = get_stack_version(component_select_name)
if stack_version_override is not None:
stack_version = stack_version_override
component_conf_dir = conf_dict
if plugin_enabled:
if api_version is not None and api_version == 'v2':
ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
policy_user,is_security_enabled,is_stack_supports_ranger_kerberos,component_user,component_user_principal,component_user_keytab)
else:
ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
policy_user)
current_datetime = datetime.now()
File(format('{component_conf_dir}/ranger-security.xml'),
owner = component_user,
group = component_group,
mode = 0644,
content = InlineTemplate(format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'))
)
Directory([os.path.join('/etc', 'ranger', repo_name), os.path.join('/etc', 'ranger', repo_name, 'policycache')],
owner = component_user,
group = component_group,
mode=0775,
create_parents = True,
cd_access = 'a'
)
for cache_service in cache_service_list:
File(os.path.join('/etc', 'ranger', repo_name, 'policycache',format('{cache_service}_{repo_name}.json')),
owner = component_user,
group = component_group,
mode = 0644
)
XmlConfig(format('ranger-{service_name}-audit.xml'),
conf_dir=component_conf_dir,
configurations=plugin_audit_properties,
configuration_attributes=plugin_audit_attributes,
owner = component_user,
group = component_group,
mode=0744)
XmlConfig(format('ranger-{service_name}-security.xml'),
conf_dir=component_conf_dir,
configurations=plugin_security_properties,
configuration_attributes=plugin_security_attributes,
owner = component_user,
group = component_group,
mode=0744)
if str(service_name).lower() == 'yarn' :
XmlConfig("ranger-policymgr-ssl-yarn.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner = component_user,
group = component_group,
mode=0744)
else :
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner = component_user,
group = component_group,
mode=0744)
#This should be done by rpm
#setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list)
setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, stack_version, credential_file,
xa_audit_db_password, ssl_truststore_password, ssl_keystore_password,
component_user, component_group, java_home)
else:
File(format('{component_conf_dir}/ranger-security.xml'),
action="delete"
)
