def copy_jdbc_connector(stack_version=None):
import params
ranger_home = params.ranger_home
driver_curl_target = '/usr/share/java/mysql-connector-java.jar'
Execute(('cp', '/usr/share/java/mysql-connector-java.jar',
os.path.join(ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
Execute(('cp', '/usr/share/java/mysql-connector-java.jar',
os.path.join(ranger_home, 'jisql', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(ranger_home, 'ews', 'lib', params.jdbc_jar_name),
mode=0644)
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties=params.config['configurations']['admin-properties'],
owner=params.unix_user,
)
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties={'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
owner=params.unix_user,
)
def copy_jdbc_connector(kms_home):
import params
if params.jdbc_jar_name is None and params.driver_curl_source.endswith("/None"):
error_message = "Error! Sorry, but we can't find jdbc driver related to {0} database to download from {1}. \
Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'".format(params.db_flavor, params.jdk_location)
Logger.error(error_message)
if params.driver_curl_source and not params.driver_curl_source.endswith("/None"):
if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar):
File(params.previous_jdbc_jar, action='delete')
driver_curl_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
mode = 0644
)
Directory(os.path.join(kms_home, 'ews', 'lib'),
mode=0755
)
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
Directory(params.jdbc_libs_dir,
cd_access="a",
create_parents=True)
Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
path=["/bin", "/usr/bin/"])
File(os.path.join(kms_home, 'ews', 'webapp', 'lib', 'sajdbc4.jar'), mode=0644)
else:
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(kms_home, 'ews', 'webapp', 'lib', params.jdbc_jar_name), mode=0644)
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = params.config['configurations']['kms-properties'],
owner = params.kms_user
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{kms_home}/ews/webapp/lib/sajdbc4.jar')},
owner = params.kms_user,
)
else:
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
owner = params.kms_user,
)
def copy_jdbc_connector(stack_version=None):
import params
if params.jdbc_jar_name is None and params.driver_curl_source.endswith("/None"):
error_message = format("{db_flavor} jdbc driver cannot be downloaded from {jdk_location}\nPlease run 'ambari-server setup --jdbc-db={db_flavor} --jdbc-driver={{path_to_jdbc}}' on ambari-server host.")
raise Fail(error_message)
if params.driver_curl_source and not params.driver_curl_source.endswith("/None"):
if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar):
File(params.previous_jdbc_jar, action='delete')
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
mode = 0644
)
ranger_home = params.ranger_home
if stack_version is not None:
ranger_home = format("{stack_root}/{stack_version}/ranger-admin")
driver_curl_target = format("{ranger_home}/ews/lib/{jdbc_jar_name}")
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(ranger_home, 'ews', 'lib', 'sajdbc4.jar'), mode=0644)
Directory(params.jdbc_libs_dir,
cd_access="a",
create_parents=True)
Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(ranger_home, 'ews', 'lib',params.jdbc_jar_name), mode=0644)
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = params.config['configurations']['admin-properties'],
owner = params.unix_user,
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{ranger_home}/ews/lib/sajdbc4.jar')},
owner = params.unix_user,
)
else:
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
owner = params.unix_user,
)
def setup_ranger_db(stack_version=None):
import params
ranger_home = params.ranger_home
if stack_version is not None:
ranger_home = format("{stack_root}/{stack_version}/ranger-admin")
copy_jdbc_connector(ranger_home)
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties={'audit_store': params.ranger_audit_source_type},
owner=params.unix_user,
)
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties={
'ranger_admin_max_heap_size': params.ranger_admin_max_heap_size
},
owner=params.unix_user,
)
env_dict = {
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home
}
if params.db_flavor.lower() == 'sqla':
env_dict = {
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home,
'LD_LIBRARY_PATH': params.ld_lib_path
}
# User wants us to setup the DB user and DB?
if params.create_db_dbuser:
Logger.info('Setting up Ranger DB and DB User')
dba_setup = format('ambari-python-wrap {ranger_home}/dba_script.py -q')
Execute(
dba_setup,
environment=env_dict,
logoutput=True,
user=params.unix_user,
)
else:
Logger.info(
'Separate DBA property not set. Assuming Ranger DB and DB User exists!'
)
db_setup = format('ambari-python-wrap {ranger_home}/db_setup.py')
Execute(
db_setup,
environment=env_dict,
logoutput=True,
user=params.unix_user,
)
def setup_ranger_admin(upgrade_type=None):
import params
check_db_connnection()
File(params.driver_curl_target, mode=0644)
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties=params.config['configurations']['admin-properties']
)
custom_config = dict()
custom_config['unix_user'] = params.unix_user
custom_config['unix_group'] = params.unix_group
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties=custom_config
)
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties={'SQL_CONNECTOR_JAR': format('{driver_curl_target}')}
)
##if db flavor == oracle - set oracle home env variable
if params.db_flavor.lower() == 'oracle' and params.oracle_home:
env_dict = {'JAVA_HOME': params.java_home, 'ORACLE_HOME': params.oracle_home,
'LD_LIBRARY_PATH': params.oracle_home}
else:
env_dict = {'JAVA_HOME': params.java_home}
setup_sh = format("cd {ranger_home} && ") + as_sudo([format('{ranger_home}/setup.sh')])
Execute(setup_sh,
environment=env_dict,
logoutput=True,
)
ModifyPropertiesFile(format("{ranger_conf}/xa_system.properties"),
properties=params.config['configurations']['ranger-site'],
)
ModifyPropertiesFile(format("{ranger_conf}/ranger_webserver.properties"),
properties=params.config['configurations']['ranger-site'],
mode=0744
)
Directory(params.admin_log_dir,
owner=params.unix_user,
group=params.unix_group
)
def copy_jdbc_connector(stack_version=None):
import params
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
mode = 0644
)
ranger_home = params.ranger_home
if stack_version is not None:
ranger_home = format("{stack_root}/{stack_version}/ranger-admin")
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(ranger_home, 'ews', 'lib', 'sajdbc4.jar'), mode=0644)
Directory(params.jdbc_libs_dir,
cd_access="a",
create_parents=True)
Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(ranger_home, 'ews', 'lib',params.jdbc_jar_name), mode=0644)
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = params.config['configurations']['admin-properties'],
owner = params.unix_user,
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{ranger_home}/ews/lib/sajdbc4.jar')},
owner = params.unix_user,
)
else:
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
owner = params.unix_user,
)
def update_password_configs():
import params
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = {'db_root_password': '******', 'db_password': '******', 'KMS_MASTER_KEY_PASSWD': '_', 'REPOSITORY_CONFIG_PASSWORD': '******'},
owner = params.kms_user,
)
def setup_usersync(upgrade_type=None):
import params
PropertiesFile(format("{usersync_home}/install.properties"),
properties = params.config['configurations']['usersync-properties'],
)
custom_config = dict()
custom_config['unix_user'] = params.unix_user
custom_config['unix_group'] = params.unix_group
ModifyPropertiesFile(format("{usersync_home}/install.properties"),
properties=custom_config
)
cmd = format("cd {usersync_home} && ") + as_sudo([format('{usersync_home}/setup.sh')])
Execute(cmd, environment={'JAVA_HOME': params.java_home}, logoutput=True)
File([params.usersync_start, params.usersync_stop],
owner = params.unix_user
)
File(params.usersync_services_file,
mode = 0755,
)
Directory(params.usersync_log_dir,
owner = params.unix_user,
group = params.unix_group
)
def setup_ranger_db(rolling_upgrade=False):
import params
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source))
Directory(params.java_share_dir, mode=0755)
if not os.path.isfile(params.driver_curl_target):
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
params.driver_curl_target),
path=["/bin", "/usr/bin/"],
not_if=format("test -f {driver_curl_target}"),
sudo=True)
ranger_home = params.ranger_home
if rolling_upgrade:
ranger_home = format("/usr/hdp/{version}/ranger-admin")
if not os.path.isfile(
os.path.join(params.ranger_home, 'ews', 'lib',
params.jdbc_jar_name)):
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
os.path.join(params.ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties=params.config['configurations']['admin-properties'])
# User wants us to setup the DB user and DB?
if params.create_db_dbuser:
Logger.info('Setting up Ranger DB and DB User')
dba_setup = format('python {ranger_home}/dba_script.py -q')
Execute(dba_setup,
environment={
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home
},
logoutput=True)
else:
Logger.info(
'Separate DBA property not set. Assuming Ranger DB and DB User exists!'
)
db_setup = format('python {ranger_home}/db_setup.py')
Execute(db_setup,
environment={
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home
},
logoutput=True)
def update_password_configs():
import params
password_configs = {'db_root_password': '******', 'db_password': '******'}
if params.stack_supports_ranger_audit_db:
password_configs['audit_db_password'] = '******'
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = password_configs,
owner = params.unix_user,
)
def setup_kms_db():
import params
if params.has_ranger_admin:
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source),
mode=0644)
Directory(params.java_share_dir, mode=0755)
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_curl_target, mode=0644)
Directory(os.path.join(params.kms_home, 'ews', 'lib'), mode=0755)
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
os.path.join(params.kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(params.kms_home, 'ews', 'webapp', 'lib',
params.jdbc_jar_name),
mode=0644)
ModifyPropertiesFile(
format("/usr/hdp/current/ranger-kms/install.properties"),
properties=params.config['configurations']['kms-properties'])
dba_setup = format('python {kms_home}/dba_script.py -q')
db_setup = format('python {kms_home}/db_setup.py')
Execute(dba_setup,
environment={
'RANGER_KMS_HOME': params.kms_home,
'JAVA_HOME': params.java_home
},
logoutput=True)
Execute(db_setup,
environment={
'RANGER_KMS_HOME': params.kms_home,
'JAVA_HOME': params.java_home
},
logoutput=True)
def setup_ranger_db(stack_version=None):
import params
ranger_home = params.ranger_home
version = params.version
if stack_version is not None:
version = stack_version
copy_jdbc_connector(stack_version=version)
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties={'audit_store': params.ranger_audit_source_type},
owner=params.unix_user,
)
env_dict = {
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home
}
# User wants us to setup the DB user and DB?
if params.create_db_dbuser:
Logger.info('Setting up Ranger DB and DB User')
dba_setup = format('ambari-python-wrap {ranger_home}/dba_script.py -q')
Execute(
dba_setup,
environment=env_dict,
logoutput=True,
user=params.unix_user,
)
else:
Logger.info(
'Separate DBA property not set. Assuming Ranger DB and DB User exists!'
)
db_setup = format('ambari-python-wrap {ranger_home}/db_setup.py')
Execute(
db_setup,
environment=env_dict,
logoutput=True,
user=params.unix_user,
)
def metadata(type='server'):
import params
# Needed by both Server and Client
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
if type == "server":
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(format('{conf_dir}/solr'),
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
File(format("{expanded_war_dir}/atlas.war"),
content=StaticFile(
format('{metadata_home}/server/webapp/atlas.war')))
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_log4j_content))
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content))
if not is_empty(params.atlas_admin_username) and not is_empty(
params.atlas_admin_password):
psswd_output = hashlib.sha256(
params.atlas_admin_password).hexdigest()
ModifyPropertiesFile(
format("{conf_dir}/users-credentials.properties"),
properties={
format('{atlas_admin_username}'):
format('ROLE_ADMIN::{psswd_output}')
},
owner=params.metadata_user)
files_to_chown = [
format("{conf_dir}/policy-store.txt"),
format("{conf_dir}/users-credentials.properties")
]
for file in files_to_chown:
if os.path.exists(file):
Execute(
('chown', format('{metadata_user}:{user_group}'), file),
sudo=True)
Execute(('chmod', '644', file), sudo=True)
if params.metadata_solrconfig_content:
File(format("{conf_dir}/solr/solrconfig.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_solrconfig_content))
# Needed by both Server and Client
PropertiesFile(format('{conf_dir}/{conf_file}'),
properties=params.application_properties,
mode=0600,
owner=params.metadata_user,
group=params.user_group)
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)
if type == 'server' and params.search_backend_solr and params.has_infra_solr:
solr_cloud_util.setup_solr_client(params.config)
check_znode()
jaasFile = params.atlas_jaas_file if params.security_enabled else None
upload_conf_set('atlas_configs', jaasFile)
if params.security_enabled: # update permissions before creating the collections
solr_cloud_util.add_solr_roles(
params.config,
roles=[
params.infra_solr_role_atlas,
params.infra_solr_role_ranger_audit,
params.infra_solr_role_dev
],
new_service_principals=[params.atlas_jaas_principal])
create_collection('vertex_index', 'atlas_configs', jaasFile)
create_collection('edge_index', 'atlas_configs', jaasFile)
create_collection('fulltext_index', 'atlas_configs', jaasFile)
if params.security_enabled:
secure_znode(format('{infra_solr_znode}/configs/atlas_configs'),
jaasFile)
secure_znode(format('{infra_solr_znode}/collections/vertex_index'),
jaasFile)
secure_znode(format('{infra_solr_znode}/collections/edge_index'),
jaasFile)
secure_znode(
format('{infra_solr_znode}/collections/fulltext_index'),
jaasFile)
File(params.atlas_hbase_setup,
group=params.user_group,
owner=params.hbase_user,
content=Template("atlas_hbase_setup.rb.j2"))
is_atlas_upgrade_support = check_stack_feature(
StackFeature.ATLAS_UPGRADE_SUPPORT,
get_stack_feature_version(params.config))
if is_atlas_upgrade_support and params.security_enabled:
File(params.atlas_kafka_setup,
group=params.user_group,
owner=params.kafka_user,
content=Template("atlas_kafka_acl.sh.j2"))
# files required only in case if kafka broker is not present on the host as configured component
if not params.host_with_kafka:
File(format("{kafka_conf_dir}/kafka-env.sh"),
owner=params.kafka_user,
content=InlineTemplate(params.kafka_env_sh_template))
File(format("{kafka_conf_dir}/kafka_jaas.conf"),
group=params.user_group,
owner=params.kafka_user,
content=Template("kafka_jaas.conf.j2"))
if params.stack_supports_atlas_hdfs_site_on_namenode_ha and len(
params.namenode_host) > 1:
XmlConfig(
"hdfs-site.xml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configurationAttributes']
['hdfs-site'],
owner=params.metadata_user,
group=params.user_group,
mode=0644)
else:
File(format('{conf_dir}/hdfs-site.xml'), action="delete")
'''
Atlas requires hadoop core-site.xml to resolve users/groups synced in HadoopUGI for
authentication and authorization process. Earlier the core-site.xml was available in
Hbase conf directory which is a part of Atlas class-path, from stack 2.6 onwards,
core-site.xml is no more available in Hbase conf directory. Hence need to create
core-site.xml in Atlas conf directory.
'''
if params.stack_supports_atlas_core_site and params.has_namenode:
XmlConfig(
"core-site.xml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configurationAttributes']
['core-site'],
owner=params.metadata_user,
group=params.user_group,
mode=0644)
Directory(
format('{metadata_home}/'),
owner=params.metadata_user,
group=params.user_group,
recursive_ownership=True,
)
def setup_ranger_db(rolling_upgrade=False):
import params
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source),
mode=0644)
Directory(params.java_share_dir, mode=0755, recursive=True, cd_access="a")
if params.db_flavor.lower() != 'sqla':
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_curl_target, mode=0644)
ranger_home = params.ranger_home
if rolling_upgrade:
ranger_home = format("/usr/hdp/{version}/ranger-admin")
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C',
params.tmp_dir),
sudo=True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive,
os.path.join(params.ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
Directory(params.jdbc_libs_dir, cd_access="a", recursive=True)
Execute(as_sudo([
'yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir
],
auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
os.path.join(params.ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(params.ranger_home, 'ews', 'lib', params.jdbc_jar_name),
mode=0644)
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties=params.config['configurations']['admin-properties'],
owner=params.unix_user,
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties={
'SQL_CONNECTOR_JAR':
format('{ranger_home}/ews/lib/{jdbc_jar_name}')
},
owner=params.unix_user,
)
env_dict = {
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home
}
if params.db_flavor.lower() == 'sqla':
env_dict = {
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home,
'LD_LIBRARY_PATH': params.ld_lib_path
}
# User wants us to setup the DB user and DB?
if params.create_db_dbuser:
Logger.info('Setting up Ranger DB and DB User')
dba_setup = format('ambari-python-wrap {ranger_home}/dba_script.py -q')
Execute(
dba_setup,
environment=env_dict,
logoutput=True,
user=params.unix_user,
)
else:
Logger.info(
'Separate DBA property not set. Assuming Ranger DB and DB User exists!'
)
db_setup = format('ambari-python-wrap {ranger_home}/db_setup.py')
Execute(
db_setup,
environment=env_dict,
logoutput=True,
user=params.unix_user,
)
def copy_jdbc_connector(stack_version=None):
import params
if params.driver_curl_source and not params.driver_curl_source.endswith(
"/None"):
if params.previous_jdbc_jar and os.path.isfile(
params.previous_jdbc_jar):
File(params.previous_jdbc_jar, action='delete')
kms_home = params.kms_home
if stack_version is not None:
kms_home = format("{stack_root}/{stack_version}/ranger-kms")
driver_curl_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source),
mode=0644)
Directory(os.path.join(kms_home, 'ews', 'lib'), mode=0755)
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C',
params.tmp_dir),
sudo=True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive,
os.path.join(kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
Directory(params.jdbc_libs_dir, cd_access="a", create_parents=True)
Execute(as_sudo([
'yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir
],
auto_escape=False),
path=["/bin", "/usr/bin/"])
File(os.path.join(kms_home, 'ews', 'webapp', 'lib', 'sajdbc4.jar'),
mode=0644)
else:
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
os.path.join(kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(kms_home, 'ews', 'webapp', 'lib',
params.jdbc_jar_name),
mode=0644)
ModifyPropertiesFile(
format("{kms_home}/install.properties"),
properties=params.config['configurations']['kms-properties'],
owner=params.kms_user)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(
format("{kms_home}/install.properties"),
properties={
'SQL_CONNECTOR_JAR':
format('{kms_home}/ews/webapp/lib/sajdbc4.jar')
},
owner=params.kms_user,
)
else:
ModifyPropertiesFile(
format("{kms_home}/install.properties"),
properties={'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
owner=params.kms_user,
)
def write_configurations(self, params, is_starting):
if os.path.isfile(params.nifi_config_dir + '/bootstrap.conf'):
bootstrap_current_conf = nifi_toolkit_util_common.convert_properties_to_dict(
params.nifi_config_dir + '/bootstrap.conf')
master_key = bootstrap_current_conf[
'nifi.bootstrap.sensitive.key'] if 'nifi.bootstrap.sensitive.key' in bootstrap_current_conf else None
else:
master_key = None
if os.path.isfile(params.nifi_config_dir + '/nifi.properties'):
nifi_current_properties = nifi_toolkit_util_common.convert_properties_to_dict(
params.nifi_config_dir + '/nifi.properties')
if 'nifi.sensitive.props.key' in nifi_current_properties and nifi_current_properties[
'nifi.sensitive.props.key']:
params.nifi_properties[
'nifi.sensitive.props.key'] = nifi_current_properties[
'nifi.sensitive.props.key']
if 'nifi.sensitive.props.key.protected' in nifi_current_properties and nifi_current_properties[
'nifi.sensitive.props.key.protected']:
params.nifi_properties[
'nifi.sensitive.props.key.protected'] = nifi_current_properties[
'nifi.sensitive.props.key.protected']
else:
nifi_current_properties = params.nifi_properties
params.nifi_toolkit_tls_regenerate = True
# Resolve and populate required security values and hashes
params.nifi_properties = nifi_toolkit_util_common.update_nifi_ssl_properties(
params.nifi_properties, params.nifi_truststore,
params.nifi_truststoreType, params.nifi_truststorePasswd,
params.nifi_keystore, params.nifi_keystoreType,
params.nifi_keystorePasswd, params.nifi_keyPasswd,
nifi_toolkit_util_common.NIFI)
# determine whether new keystore/truststore should be regenerated
run_tls = (params.nifi_ca_host and params.nifi_ssl_enabled) and (
params.nifi_toolkit_tls_regenerate
or nifi_toolkit_util_common.generate_keystore_truststore(
nifi_current_properties, params.nifi_properties, master_key,
nifi_toolkit_util_common.NIFI))
if run_tls:
nifi_toolkit_util_common.move_keystore_truststore(
nifi_current_properties, nifi_toolkit_util_common.NIFI)
params.nifi_properties = nifi_toolkit_util_common.create_keystore_truststore(
params.nifi_properties, is_starting,
params.nifi_toolkit_java_options, params.nifi_config_dir,
params.nifi_user, params.nifi_group,
nifi_toolkit_util_common.NIFI)
elif not params.nifi_ssl_enabled:
params.nifi_properties = nifi_toolkit_util_common.clean_toolkit_client_files(
nifi_current_properties, params.nifi_properties,
nifi_toolkit_util_common.NIFI)
elif params.nifi_ssl_enabled and not run_tls and os.path.isfile(
params.nifi_config_dir + '/nifi.properties'):
params.nifi_properties = nifi_toolkit_util_common.populate_ssl_properties(
nifi_toolkit_util_common.convert_properties_to_dict(
params.nifi_config_dir + '/nifi.properties'),
params.nifi_properties, params, nifi_toolkit_util_common.NIFI)
# if this is an additional node being added to an existing cluster do not include the node identity information
if params.is_additional_node:
Logger.info(
"Excluding initial admin and node identity section from authorizers due to existing cluster"
)
params.nifi_authorizers_content = params.nifi_authorizers_content.replace(
'{{nifi_ssl_config_content | replace("Node","Initial User")}}',
'')
params.nifi_authorizers_content = params.nifi_authorizers_content.replace(
'{{nifi_ssl_config_content}}', '')
params.nifi_authorizers_content = params.nifi_authorizers_content.replace(
'{{nifi_initial_admin_id}}', '')
# Write configuration files
self.write_files(params)
# Encrypt files
nifi_toolkit_util_common.encrypt_sensitive_properties(
params.nifi_config_dir,
params.jdk64_home,
params.nifi_toolkit_java_options,
params.nifi_user,
master_key,
params.nifi_security_encrypt_configuration_password,
is_starting,
params.toolkit_tmp_dir,
params.stack_version_buildnum,
nifi_toolkit_util_common.NIFI,
nifi_flow_config_dir=params.nifi_flow_config_dir,
nifi_sensitive_props_key=params.nifi_sensitive_props_key,
support_encrypt_authorizers=params.
stack_support_encrypt_authorizers)
# Apply Hashed Ambari parameters by retrieving new master key and hashing required parameters for Ambari
bootstrap_current_conf = nifi_toolkit_util_common.convert_properties_to_dict(
format("{params.nifi_bootstrap_file}"))
new_master_key = bootstrap_current_conf[
'nifi.bootstrap.sensitive.key'] if 'nifi.bootstrap.sensitive.key' in bootstrap_current_conf else None
if new_master_key:
nifi_hashed_params = nifi_toolkit_util_common.update_nifi_ambari_hash_properties(
params.nifi_truststorePasswd, params.nifi_keystorePasswd,
params.nifi_keyPasswd, new_master_key,
nifi_toolkit_util_common.NIFI)
ModifyPropertiesFile(
format("{params.nifi_config_dir}/nifi.properties"),
properties=nifi_hashed_params,
owner=params.nifi_user)
else:
raise Fail(
"Unable to persist ambari hashes due to no master key! Please validate this was written to bootstrap.conf file."
)
def setup_kms_db():
import params
if params.has_ranger_admin:
password_validation(params.kms_master_key_password, 'KMS master key')
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
mode = 0644
)
Directory(params.java_share_dir,
mode=0755,
recursive=True,
cd_access="a"
)
if params.db_flavor.lower() != 'sqla':
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_curl_target, mode=0644)
Directory(os.path.join(params.kms_home, 'ews', 'lib'),
mode=0755
)
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(params.kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
Directory(params.jdbc_libs_dir,
cd_access="a",
recursive=True)
Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(params.kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(params.kms_home, 'ews', 'webapp', 'lib', params.jdbc_jar_name), mode=0644)
ModifyPropertiesFile(format("/usr/hdp/current/ranger-kms/install.properties"),
properties = params.config['configurations']['kms-properties'],
owner = params.kms_user
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{kms_home}/ews/webapp/lib/{jdbc_jar_name}')},
owner = params.kms_user,
)
env_dict = {'RANGER_KMS_HOME':params.kms_home, 'JAVA_HOME': params.java_home}
if params.db_flavor.lower() == 'sqla':
env_dict = {'RANGER_KMS_HOME':params.kms_home, 'JAVA_HOME': params.java_home, 'LD_LIBRARY_PATH':params.ld_library_path}
dba_setup = format('python {kms_home}/dba_script.py -q')
db_setup = format('python {kms_home}/db_setup.py')
if params.create_db_user:
Logger.info('Setting up Ranger KMS DB and DB User')
Execute(dba_setup, environment=env_dict, logoutput=True, user=params.kms_user, tries=5, try_sleep=10)
else:
Logger.info('Separate DBA property not set. Assuming Ranger KMS DB and DB User exists!')
Execute(db_setup, environment=env_dict, logoutput=True, user=params.kms_user, tries=5, try_sleep=10)
def setup_ranger_db(stack_version=None):
import params
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source),
mode=0644)
Directory(params.java_share_dir,
mode=0755,
create_parents=True,
cd_access="a")
if params.db_flavor.lower() != 'sqla':
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_curl_target, mode=0644)
ranger_home = params.ranger_home
version = params.version
if stack_version is not None:
ranger_home = format("{stack_root}/{stack_version}/ranger-admin")
version = stack_version
copy_jdbc_connector(stack_version=version)
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties=params.config['configurations']['admin-properties'],
owner=params.unix_user,
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties={
'SQL_CONNECTOR_JAR':
format('{ranger_home}/ews/lib/{jdbc_jar_name}')
},
owner=params.unix_user,
)
env_dict = {
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home
}
if params.db_flavor.lower() == 'sqla':
env_dict = {
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home,
'LD_LIBRARY_PATH': params.ld_lib_path
}
# User wants us to setup the DB user and DB?
if params.create_db_dbuser:
Logger.info('Setting up Ranger DB and DB User')
dba_setup = format('python {ranger_home}/dba_script.py -q')
Execute(
dba_setup,
environment=env_dict,
logoutput=True,
user=params.unix_user,
)
else:
Logger.info(
'Separate DBA property not set. Assuming Ranger DB and DB User exists!'
)
db_setup = format('python {ranger_home}/db_setup.py')
Execute(
db_setup,
environment=env_dict,
logoutput=True,
user=params.unix_user,
)
def write_configurations(self, params, is_starting):
if os.path.isfile(params.nifi_registry_config_dir + '/bootstrap.conf'):
bootstrap_current_conf = nifi_toolkit_util_common.convert_properties_to_dict(
params.nifi_registry_config_dir + '/bootstrap.conf')
master_key = bootstrap_current_conf[
'nifi.registry.bootstrap.sensitive.key'] if 'nifi.registry.bootstrap.sensitive.key' in bootstrap_current_conf else None
else:
master_key = None
if os.path.isfile(params.nifi_registry_config_dir +
'/nifi-registry.properties'):
nifi_registry_current_properties = nifi_toolkit_util_common.convert_properties_to_dict(
params.nifi_registry_config_dir + '/nifi-registry.properties')
if 'nifi.registry.sensitive.props.key' in nifi_registry_current_properties and nifi_registry_current_properties[
'nifi.registry.sensitive.props.key']:
params.nifi_registry_properties[
'nifi.registry.sensitive.props.key'] = nifi_registry_current_properties[
'nifi.registry.sensitive.props.key']
if 'nifi.registry.sensitive.props.key.protected' in nifi_registry_current_properties and nifi_registry_current_properties[
'nifi.registry.sensitive.props.key.protected']:
params.nifi_registry_properties[
'nifi.registry.sensitive.props.key.protected'] = nifi_registry_current_properties[
'nifi.registry.sensitive.props.key.protected']
else:
nifi_registry_current_properties = params.nifi_registry_properties
params.nifi_toolkit_tls_regenerate = True
#resolve and populate required security values and hashes
params.nifi_registry_properties = nifi_toolkit_util_common.update_nifi_ssl_properties(
params.nifi_registry_properties, params.nifi_registry_truststore,
params.nifi_registry_truststoreType,
params.nifi_registry_truststorePasswd,
params.nifi_registry_keystore, params.nifi_registry_keystoreType,
params.nifi_registry_keystorePasswd,
params.nifi_registry_keyPasswd,
nifi_toolkit_util_common.NIFI_REGISTRY)
#determine whether new keystore/truststore should be regenerated
run_tls = (params.nifi_ca_host
and params.nifi_registry_ssl_enabled) and (
params.nifi_toolkit_tls_regenerate or
nifi_toolkit_util_common.generate_keystore_truststore(
nifi_registry_current_properties,
params.nifi_registry_properties, master_key,
nifi_toolkit_util_common.NIFI_REGISTRY))
if run_tls:
nifi_toolkit_util_common.move_keystore_truststore(
nifi_registry_current_properties,
nifi_toolkit_util_common.NIFI_REGISTRY)
params.nifi_registry_properties = nifi_toolkit_util_common.create_keystore_truststore(
params.nifi_registry_properties, is_starting,
params.nifi_toolkit_java_options,
params.nifi_registry_config_dir, params.nifi_registry_user,
params.nifi_registry_group,
nifi_toolkit_util_common.NIFI_REGISTRY)
elif not params.nifi_registry_ssl_enabled:
params.nifi_registry_properties = nifi_toolkit_util_common.clean_toolkit_client_files(
nifi_registry_current_properties,
params.nifi_registry_properties,
nifi_toolkit_util_common.NIFI_REGISTRY)
elif params.nifi_registry_ssl_enabled and not run_tls and os.path.isfile(
params.nifi_registry_config_dir + '/nifi-registry.properties'):
params.nifi_registry_properties = nifi_toolkit_util_common.populate_ssl_properties(
nifi_toolkit_util_common.convert_properties_to_dict(
params.nifi_registry_config_dir +
'/nifi-registry.properties'),
params.nifi_registry_properties, params,
nifi_toolkit_util_common.NIFI_REGISTRY)
self.write_files(params)
nifi_toolkit_util_common.encrypt_sensitive_properties(
params.nifi_registry_config_dir, params.jdk64_home,
params.nifi_toolkit_java_options, params.nifi_registry_user,
master_key,
params.nifi_registry_security_encrypt_configuration_password,
is_starting, params.toolkit_tmp_dir, params.stack_version_buildnum,
nifi_toolkit_util_common.NIFI_REGISTRY)
#Apply Hashed Ambari parameters by retrieving new master key and hashing required parameters for Ambari
bootstrap_current_conf = nifi_toolkit_util_common.convert_properties_to_dict(
format("{params.nifi_registry_bootstrap_file}"))
master_key = bootstrap_current_conf[
'nifi.registry.bootstrap.sensitive.key'] if 'nifi.registry.bootstrap.sensitive.key' in bootstrap_current_conf else None
if master_key:
nifi_registry_hashed_params = nifi_toolkit_util_common.update_nifi_ambari_hash_properties(
params.nifi_registry_truststorePasswd,
params.nifi_registry_keystorePasswd,
params.nifi_registry_keyPasswd, master_key,
nifi_toolkit_util_common.NIFI_REGISTRY)
ModifyPropertiesFile(format(
"{params.nifi_registry_config_dir}/nifi-registry.properties"),
properties=nifi_registry_hashed_params,
owner=params.nifi_registry_user)
else:
raise Fail(
"Unable to persist ambari hashes due to no master key! Please validate this was written to bootstrap.conf file."
)
def copy_jdbc_connector(stack_version=None):
import params
kms_home = params.kms_home
if stack_version is not None:
kms_home = format("/usr/hdp/{stack_version}/ranger-kms")
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source),
mode=0644)
Directory(params.java_share_dir, mode=0755, recursive=True, cd_access="a")
if params.db_flavor.lower() != 'sqla':
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_curl_target, mode=0644)
Directory(os.path.join(kms_home, 'ews', 'lib'), mode=0755)
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C',
params.tmp_dir),
sudo=True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive,
os.path.join(kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
Directory(params.jdbc_libs_dir, cd_access="a", recursive=True)
Execute(as_sudo([
'yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir
],
auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
os.path.join(kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(kms_home, 'ews', 'webapp', 'lib', params.jdbc_jar_name),
mode=0644)
ModifyPropertiesFile(
format("{kms_home}/install.properties"),
properties=params.config['configurations']['kms-properties'],
owner=params.kms_user)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(
format("{kms_home}/install.properties"),
properties={
'SQL_CONNECTOR_JAR':
format('{kms_home}/ews/webapp/lib/{jdbc_jar_name}')
},
owner=params.kms_user,
)
def setup_ranger_admin(upgrade_type=None):
import params
check_db_connnection()
if params.driver_curl_source and not params.driver_curl_source.endswith("/None"):
if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar):
File(params.previous_jdbc_jar, action='delete')
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
mode = 0644
)
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_curl_target, mode=0644)
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = params.config['configurations']['admin-properties']
)
custom_config = dict()
custom_config['unix_user'] = params.unix_user
custom_config['unix_group'] = params.unix_group
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties=custom_config
)
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')}
)
##if db flavor == oracle - set oracle home env variable
if params.db_flavor.lower() == 'oracle' and params.oracle_home:
env_dict = {'JAVA_HOME': params.java_home, 'ORACLE_HOME':params.oracle_home, 'LD_LIBRARY_PATH':params.oracle_home}
else:
env_dict = {'JAVA_HOME': params.java_home}
setup_sh = format("cd {ranger_home} && ") + as_sudo([format('{ranger_home}/setup.sh')])
Execute(setup_sh,
environment=env_dict,
logoutput=True,
)
ModifyPropertiesFile(format("{ranger_conf}/xa_system.properties"),
properties = params.config['configurations']['ranger-site'],
)
ModifyPropertiesFile(format("{ranger_conf}/ranger_webserver.properties"),
properties = params.config['configurations']['ranger-site'],
mode=0744
)
Directory(params.admin_log_dir,
owner = params.unix_user,
group = params.unix_group
)
def metadata(type='server'):
import params
# Needed by both Server and Client
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
if type == "server":
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(format('{conf_dir}/solr'),
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
File(format("{expanded_war_dir}/atlas.war"),
content=StaticFile(
format('{metadata_home}/server/webapp/atlas.war')))
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_log4j_content))
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0644,
content=InlineTemplate(params.metadata_env_content))
if not is_empty(params.atlas_admin_username) and not is_empty(
params.atlas_admin_password):
psswd_output = hashlib.sha256(
params.atlas_admin_password).hexdigest()
ModifyPropertiesFile(
format("{conf_dir}/users-credentials.properties"),
properties={
format('{atlas_admin_username}'):
format('ROLE_ADMIN::{psswd_output}')
},
owner=params.metadata_user)
files_to_chown = [
format("{conf_dir}/atlas-simple-authz-policy.json"),
format("{conf_dir}/users-credentials.properties")
]
for file in files_to_chown:
if os.path.exists(file):
Execute(
('chown', format('{metadata_user}:{user_group}'), file),
sudo=True)
Execute(('chmod', '640', file), sudo=True)
if params.metadata_solrconfig_content:
File(format("{conf_dir}/solr/solrconfig.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_solrconfig_content))
generate_logfeeder_input_config(
'atlas',
Template("input.config-atlas.json.j2", extra_imports=[default]))
# Needed by both Server and Client
PropertiesFile(format('{conf_dir}/{conf_file}'),
properties=params.application_properties,
mode=0600,
owner=params.metadata_user,
group=params.user_group)
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)
if type == 'server' and params.search_backend_solr and params.has_infra_solr:
solr_cloud_util.setup_solr_client(params.config)
check_znode()
jaasFile = params.atlas_jaas_file if params.security_enabled else None
upload_conf_set('atlas_configs', jaasFile)
if params.security_enabled: # update permissions before creating the collections
solr_cloud_util.add_solr_roles(
params.config,
roles=[
params.infra_solr_role_atlas,
params.infra_solr_role_ranger_audit,
params.infra_solr_role_dev
],
new_service_principals=[params.atlas_jaas_principal])
create_collection('vertex_index', 'atlas_configs', jaasFile)
create_collection('edge_index', 'atlas_configs', jaasFile)
create_collection('fulltext_index', 'atlas_configs', jaasFile)
if params.security_enabled:
secure_znode(format('{infra_solr_znode}/configs/atlas_configs'),
jaasFile)
secure_znode(format('{infra_solr_znode}/collections/vertex_index'),
jaasFile)
secure_znode(format('{infra_solr_znode}/collections/edge_index'),
jaasFile)
secure_znode(
format('{infra_solr_znode}/collections/fulltext_index'),
jaasFile)
File(params.atlas_hbase_setup,
group=params.user_group,
owner=params.hbase_user,
content=Template("atlas_hbase_setup.rb.j2"))
is_atlas_upgrade_support = True
if is_atlas_upgrade_support and params.security_enabled:
File(params.atlas_kafka_setup,
group=params.user_group,
owner=params.kafka_user,
content=Template("atlas_kafka_acl.sh.j2"))
# files required only in case if kafka broker is not present on the host as configured component
if not params.host_with_kafka:
File(format("{kafka_conf_dir}/kafka-env.sh"),
owner=params.kafka_user,
content=InlineTemplate(params.kafka_env_sh_template))
File(format("{kafka_conf_dir}/kafka_jaas.conf"),
group=params.user_group,
owner=params.kafka_user,
content=Template("kafka_jaas.conf.j2"))
if params.stack_supports_atlas_hdfs_site_on_namenode_ha and len(
params.namenode_host) > 1:
XmlConfig(
"hdfs-site.xml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configurationAttributes']
['hdfs-site'],
owner=params.metadata_user,
group=params.user_group,
mode=0644)
else:
File(format('{conf_dir}/hdfs-site.xml'), action="delete")
if params.stack_supports_atlas_core_site and params.has_namenode:
XmlConfig(
"core-site.xml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configurationAttributes']
['core-site'],
owner=params.metadata_user,
group=params.user_group,
mode=0644,
xml_include_file=params.mount_table_xml_inclusion_file_full_path)
if params.mount_table_content:
File(params.mount_table_xml_inclusion_file_full_path,
owner=params.metadata_user,
group=params.user_group,
content=params.mount_table_content,
mode=0644)
Directory(
format('{metadata_home}/'),
owner=params.metadata_user,
group=params.user_group,
recursive_ownership=True,
)
