def start(self, env, upgrade_type=None):
import params
env.set_params(params)
self.configure(env, upgrade_type=upgrade_type)
if params.stack_supports_infra_client and params.audit_solr_enabled and params.is_solrCloud_enabled:
solr_cloud_util.setup_solr_client(params.config,
custom_log4j=params.custom_log4j)
setup_ranger_audit_solr()
ranger_service('ranger_admin')
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
# setup db only if in case stack version is > 2.6
self.configure(env, upgrade_type=upgrade_type, setup_db=params.stack_supports_ranger_setup_db_on_start)
if params.stack_supports_infra_client and params.audit_solr_enabled and params.is_solrCloud_enabled:
solr_cloud_util.setup_solr_client(params.config, custom_log4j = params.custom_log4j)
setup_ranger_audit_solr()
ranger_service('ranger_admin')
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
self.install_ranger()
Execute(
"chown -R %s:%s %s" %
(params.unix_user, params.unix_group, params.install_dir_admin))
# setup db only if in case HDP version is > 2.6
self.configure(env,
upgrade_type=upgrade_type,
setup_db=params.stack_supports_ranger_setup_db_on_start)
if params.stack_supports_infra_client and params.audit_solr_enabled and params.is_solrCloud_enabled:
solr_cloud_util.setup_solr_client(params.config,
custom_log4j=params.custom_log4j)
setup_ranger_audit_solr()
ranger_service('ranger_admin')
def setup_infra_solr(name=None):
import params
if name == 'server':
Directory([
params.infra_solr_log_dir, params.infra_solr_piddir,
params.infra_solr_datadir, params.infra_solr_data_resources_dir
],
mode=0755,
cd_access='a',
create_parents=True,
owner=params.infra_solr_user,
group=params.user_group)
Directory([params.solr_dir, params.infra_solr_conf],
mode=0755,
cd_access='a',
owner=params.infra_solr_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True)
File(params.infra_solr_log,
mode=0644,
owner=params.infra_solr_user,
group=params.user_group,
content='')
File(format("{infra_solr_conf}/infra-solr-env.sh"),
content=InlineTemplate(params.solr_env_content),
mode=0755,
owner=params.infra_solr_user,
group=params.user_group)
File(format("{infra_solr_datadir}/solr.xml"),
content=InlineTemplate(params.solr_xml_content),
owner=params.infra_solr_user,
group=params.user_group)
File(format("{infra_solr_conf}/log4j2.xml"),
content=InlineTemplate(params.solr_log4j_content),
owner=params.infra_solr_user,
group=params.user_group)
custom_security_json_location = format(
"{infra_solr_conf}/custom-security.json")
File(custom_security_json_location,
content=InlineTemplate(params.infra_solr_security_json_content),
owner=params.infra_solr_user,
group=params.user_group,
mode=0640)
if params.security_enabled:
File(format("{infra_solr_jaas_file}"),
content=Template("infra_solr_jaas.conf.j2"),
owner=params.infra_solr_user)
File(format("{infra_solr_conf}/security.json"),
content=Template("infra-solr-security.json.j2"),
owner=params.infra_solr_user,
group=params.user_group,
mode=0640)
if os.path.exists(params.limits_conf_dir):
File(os.path.join(params.limits_conf_dir, 'infra-solr.conf'),
owner='root',
group='root',
mode=0644,
content=Template("infra-solr.conf.j2"))
elif name == 'client':
solr_cloud_util.setup_solr_client(params.config)
else:
raise Fail('Nor client or server were selected to install.')
def metadata(type='server'):
import params
# Needed by both Server and Client
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
if type == "server":
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(format('{conf_dir}/solr'),
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
File(format("{expanded_war_dir}/atlas.war"),
content=StaticFile(
format('{metadata_home}/server/webapp/atlas.war')))
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_log4j_content))
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content))
if not is_empty(params.atlas_admin_username) and not is_empty(
params.atlas_admin_password):
psswd_output = hashlib.sha256(
params.atlas_admin_password).hexdigest()
ModifyPropertiesFile(
format("{conf_dir}/users-credentials.properties"),
properties={
format('{atlas_admin_username}'):
format('ROLE_ADMIN::{psswd_output}')
},
owner=params.metadata_user)
files_to_chown = [
format("{conf_dir}/policy-store.txt"),
format("{conf_dir}/users-credentials.properties")
]
for file in files_to_chown:
if os.path.exists(file):
Execute(
('chown', format('{metadata_user}:{user_group}'), file),
sudo=True)
Execute(('chmod', '644', file), sudo=True)
if params.metadata_solrconfig_content:
File(format("{conf_dir}/solr/solrconfig.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_solrconfig_content))
# Needed by both Server and Client
PropertiesFile(format('{conf_dir}/{conf_file}'),
properties=params.application_properties,
mode=0600,
owner=params.metadata_user,
group=params.user_group)
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)
if type == 'server' and params.search_backend_solr and params.has_infra_solr:
solr_cloud_util.setup_solr_client(params.config)
check_znode()
jaasFile = params.atlas_jaas_file if params.security_enabled else None
upload_conf_set('atlas_configs', jaasFile)
if params.security_enabled: # update permissions before creating the collections
solr_cloud_util.add_solr_roles(
params.config,
roles=[
params.infra_solr_role_atlas,
params.infra_solr_role_ranger_audit,
params.infra_solr_role_dev
],
new_service_principals=[params.atlas_jaas_principal])
create_collection('vertex_index', 'atlas_configs', jaasFile)
create_collection('edge_index', 'atlas_configs', jaasFile)
create_collection('fulltext_index', 'atlas_configs', jaasFile)
if params.security_enabled:
secure_znode(format('{infra_solr_znode}/configs/atlas_configs'),
jaasFile)
secure_znode(format('{infra_solr_znode}/collections/vertex_index'),
jaasFile)
secure_znode(format('{infra_solr_znode}/collections/edge_index'),
jaasFile)
secure_znode(
format('{infra_solr_znode}/collections/fulltext_index'),
jaasFile)
File(params.atlas_hbase_setup,
group=params.user_group,
owner=params.hbase_user,
content=Template("atlas_hbase_setup.rb.j2"))
is_atlas_upgrade_support = check_stack_feature(
StackFeature.ATLAS_UPGRADE_SUPPORT,
get_stack_feature_version(params.config))
if is_atlas_upgrade_support and params.security_enabled:
File(params.atlas_kafka_setup,
group=params.user_group,
owner=params.kafka_user,
content=Template("atlas_kafka_acl.sh.j2"))
# files required only in case if kafka broker is not present on the host as configured component
if not params.host_with_kafka:
File(format("{kafka_conf_dir}/kafka-env.sh"),
owner=params.kafka_user,
content=InlineTemplate(params.kafka_env_sh_template))
File(format("{kafka_conf_dir}/kafka_jaas.conf"),
group=params.user_group,
owner=params.kafka_user,
content=Template("kafka_jaas.conf.j2"))
if params.stack_supports_atlas_hdfs_site_on_namenode_ha and len(
params.namenode_host) > 1:
XmlConfig(
"hdfs-site.xml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configurationAttributes']
['hdfs-site'],
owner=params.metadata_user,
group=params.user_group,
mode=0644)
else:
File(format('{conf_dir}/hdfs-site.xml'), action="delete")
'''
Atlas requires hadoop core-site.xml to resolve users/groups synced in HadoopUGI for
authentication and authorization process. Earlier the core-site.xml was available in
Hbase conf directory which is a part of Atlas class-path, from stack 2.6 onwards,
core-site.xml is no more available in Hbase conf directory. Hence need to create
core-site.xml in Atlas conf directory.
'''
if params.stack_supports_atlas_core_site and params.has_namenode:
XmlConfig(
"core-site.xml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configurationAttributes']
['core-site'],
owner=params.metadata_user,
group=params.user_group,
mode=0644)
Directory(
format('{metadata_home}/'),
owner=params.metadata_user,
group=params.user_group,
recursive_ownership=True,
)
def setup_infra_solr(name=None):
import params
if name == 'server':
Directory([
params.infra_solr_log_dir, params.infra_solr_piddir,
params.infra_solr_datadir, params.infra_solr_data_resources_dir
],
mode=0755,
cd_access='a',
create_parents=True,
owner=params.infra_solr_user,
group=params.user_group)
Directory([params.solr_dir, params.infra_solr_conf],
mode=0755,
cd_access='a',
owner=params.infra_solr_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True)
File(params.infra_solr_log,
mode=0644,
owner=params.infra_solr_user,
group=params.user_group,
content='')
File(format("{infra_solr_conf}/infra-solr-env.sh"),
content=InlineTemplate(params.solr_env_content),
mode=0755,
owner=params.infra_solr_user,
group=params.user_group)
File(format("{infra_solr_datadir}/solr.xml"),
content=InlineTemplate(params.solr_xml_content),
owner=params.infra_solr_user,
group=params.user_group)
File(format("{infra_solr_conf}/log4j.properties"),
content=InlineTemplate(params.solr_log4j_content),
owner=params.infra_solr_user,
group=params.user_group)
custom_security_json_location = format(
"{infra_solr_conf}/custom-security.json")
File(custom_security_json_location,
content=InlineTemplate(params.infra_solr_security_json_content),
owner=params.infra_solr_user,
group=params.user_group,
mode=0640)
jaas_file = params.infra_solr_jaas_file if params.security_enabled else None
url_scheme = 'https' if params.infra_solr_ssl_enabled else 'http'
create_ambari_solr_znode()
security_json_file_location = custom_security_json_location \
if params.infra_solr_security_json_content and str(params.infra_solr_security_json_content).strip() \
else format("{infra_solr_conf}/security.json") # security.json file to upload
if params.security_enabled:
File(format("{infra_solr_jaas_file}"),
content=Template("infra_solr_jaas.conf.j2"),
owner=params.infra_solr_user)
File(format("{infra_solr_conf}/security.json"),
content=Template("infra-solr-security.json.j2"),
owner=params.infra_solr_user,
group=params.user_group,
mode=0640)
solr_cloud_util.set_cluster_prop(
zookeeper_quorum=params.zookeeper_quorum,
solr_znode=params.infra_solr_znode,
java64_home=params.java64_home,
prop_name="urlScheme",
prop_value=url_scheme,
jaas_file=jaas_file)
solr_cloud_util.setup_kerberos_plugin(
zookeeper_quorum=params.zookeeper_quorum,
solr_znode=params.infra_solr_znode,
jaas_file=jaas_file,
java64_home=params.java64_home,
secure=params.security_enabled,
security_json_location=security_json_file_location)
if params.security_enabled:
solr_cloud_util.secure_solr_znode(
zookeeper_quorum=params.zookeeper_quorum,
solr_znode=params.infra_solr_znode,
jaas_file=jaas_file,
java64_home=params.java64_home,
sasl_users_str=params.infra_solr_sasl_user)
elif name == 'client':
solr_cloud_util.setup_solr_client(params.config)
else:
raise Fail('Nor client or server were selected to install.')
def metadata(type='server'):
import params
# Needed by both Server and Client
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
if type == "server":
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(format('{conf_dir}/solr'),
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
File(format("{expanded_war_dir}/atlas.war"),
content=StaticFile(
format('{metadata_home}/server/webapp/atlas.war')))
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_log4j_content))
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content))
if params.metadata_solrconfig_content:
File(format("{conf_dir}/solr/solrconfig.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_solrconfig_content))
# Needed by both Server and Client
PropertiesFile(format('{conf_dir}/{conf_file}'),
properties=params.application_properties,
mode=0644,
owner=params.metadata_user,
group=params.user_group)
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)
if type == 'server' and params.search_backend_solr and params.has_infra_solr:
solr_cloud_util.setup_solr_client(params.config)
check_znode()
jaasFile = params.atlas_jaas_file if params.security_enabled else None
upload_conf_set('atlas_configs', jaasFile)
create_collection('vertex_index', 'atlas_configs', jaasFile)
create_collection('edge_index', 'atlas_configs', jaasFile)
create_collection('fulltext_index', 'atlas_configs', jaasFile)
File(params.atlas_hbase_setup,
group=params.user_group,
owner=params.hbase_user,
content=Template("atlas_hbase_setup.rb.j2"))
if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, get_stack_feature_version(params.config)) and\
params.security_enabled and not params.host_with_kafka:
File(params.atlas_kafka_setup,
group=params.user_group,
owner=params.kafka_user,
content=Template("atlas_kafka_acl.sh.j2"))
File(format("{kafka_conf_dir}/kafka-env.sh"),
owner=params.kafka_user,
content=InlineTemplate(params.kafka_env_sh_template))
File(format("{kafka_conf_dir}/kafka_jaas.conf"),
group=params.user_group,
owner=params.kafka_user,
content=Template("kafka_jaas.conf.j2"))
def metadata(type='server'):
import params
# Needed by both Server and Client
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
if type == "server":
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(format('{conf_dir}/solr'),
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
File(format("{expanded_war_dir}/atlas.war"),
content=StaticFile(
format('{metadata_home}/server/webapp/atlas.war')))
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_log4j_content))
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content))
files_to_chown = [
format("{conf_dir}/policy-store.txt"),
format("{conf_dir}/users-credentials.properties")
]
for file in files_to_chown:
if os.path.exists(file):
Execute(
('chown', format('{metadata_user}:{user_group}'), file),
sudo=True)
Execute(('chmod', '644', file), sudo=True)
if params.metadata_solrconfig_content:
File(format("{conf_dir}/solr/solrconfig.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_solrconfig_content))
# Needed by both Server and Client
PropertiesFile(format('{conf_dir}/{conf_file}'),
properties=params.application_properties,
mode=0644,
owner=params.metadata_user,
group=params.user_group)
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)
if type == 'server' and params.search_backend_solr and params.has_infra_solr:
solr_cloud_util.setup_solr_client(params.config)
check_znode()
jaasFile = params.atlas_jaas_file if params.security_enabled else None
upload_conf_set('atlas_configs', jaasFile)
create_collection('vertex_index', 'atlas_configs', jaasFile)
create_collection('edge_index', 'atlas_configs', jaasFile)
create_collection('fulltext_index', 'atlas_configs', jaasFile)
File(params.atlas_hbase_setup,
group=params.user_group,
owner=params.hbase_user,
content=Template("atlas_hbase_setup.rb.j2"))
is_atlas_upgrade_support = check_stack_feature(
StackFeature.ATLAS_UPGRADE_SUPPORT,
get_stack_feature_version(params.config))
if is_atlas_upgrade_support and params.security_enabled:
File(params.atlas_kafka_setup,
group=params.user_group,
owner=params.kafka_user,
content=Template("atlas_kafka_acl.sh.j2"))
# files required only in case if kafka broker is not present on the host as configured component
if not params.host_with_kafka:
File(format("{kafka_conf_dir}/kafka-env.sh"),
owner=params.kafka_user,
content=InlineTemplate(params.kafka_env_sh_template))
File(format("{kafka_conf_dir}/kafka_jaas.conf"),
group=params.user_group,
owner=params.kafka_user,
content=Template("kafka_jaas.conf.j2"))
if params.stack_supports_atlas_hdfs_site_on_namenode_ha and len(
params.namenode_host) > 1:
XmlConfig(
"hdfs-site.xml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configuration_attributes']
['hdfs-site'],
owner=params.metadata_user,
group=params.user_group,
mode=0644)
else:
File(format('{conf_dir}/hdfs-site.xml'), action="delete")
def metadata(type='server'):
import params
# Needed by both Server and Client
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
if type == "server":
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(format('{conf_dir}/solr'),
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
File(format("{expanded_war_dir}/atlas.war"),
content=StaticFile(
format('{metadata_home}/server/webapp/atlas.war')))
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_log4j_content))
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content))
File(format("{conf_dir}/solr/solrconfig.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_solrconfig_content))
# Needed by both Server and Client
PropertiesFile(format('{conf_dir}/{conf_file}'),
properties=params.application_properties,
mode=0644,
owner=params.metadata_user,
group=params.user_group)
if type == 'server' and params.search_backend_solr and params.has_logsearch_solr:
solr_cloud_util.setup_solr_client(params.config)
random_num = random.random()
upload_conf_set('basic_configs', random_num)
create_collection('vertex_index', 'basic_configs')
create_collection('edge_index', 'basic_configs')
create_collection('fulltext_index', 'basic_configs')
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)
def setup_logsearch_solr(name=None):
import params
if name == 'server':
Directory([
params.logsearch_solr_log_dir, params.logsearch_solr_piddir,
params.logsearch_solr_datadir,
params.logsearch_solr_data_resources_dir
],
mode=0755,
cd_access='a',
create_parents=True,
owner=params.logsearch_solr_user,
group=params.user_group)
Directory([params.solr_dir, params.logsearch_solr_conf],
mode=0755,
cd_access='a',
owner=params.logsearch_solr_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True)
File(params.logsearch_solr_log,
mode=0644,
owner=params.logsearch_solr_user,
group=params.user_group,
content='')
File(format("{logsearch_solr_conf}/logsearch-solr-env.sh"),
content=InlineTemplate(params.solr_env_content),
mode=0755,
owner=params.logsearch_solr_user,
group=params.user_group)
File(format("{logsearch_solr_datadir}/solr.xml"),
content=InlineTemplate(params.solr_xml_content),
owner=params.logsearch_solr_user,
group=params.user_group)
File(format("{logsearch_solr_conf}/log4j.properties"),
content=InlineTemplate(params.solr_log4j_content),
owner=params.logsearch_solr_user,
group=params.user_group)
File(format("{logsearch_solr_datadir}/zoo.cfg"),
content=Template("zoo.cfg.j2"),
owner=params.logsearch_solr_user,
group=params.user_group)
zk_cli_prefix = format(
'export JAVA_HOME={java64_home}; {cloud_scripts}/zkcli.sh -zkhost {zookeeper_quorum}'
)
create_ambari_solr_znode(zk_cli_prefix)
url_scheme = 'https' if params.logsearch_solr_ssl_enabled else 'http'
Execute(format(
'{zk_cli_prefix}{logsearch_solr_znode} -cmd clusterprop -name urlScheme -val {url_scheme}'
),
user=params.logsearch_solr_user)
if params.security_enabled:
File(format("{logsearch_solr_jaas_file}"),
content=Template("logsearch_solr_jaas.conf.j2"),
owner=params.logsearch_solr_user)
security_content = '\'{"authentication":{"class": "org.apache.solr.security.KerberosPlugin"}}\''
else:
security_content = '\'{}\''
Execute(format(
'{zk_cli_prefix} -cmd put {logsearch_solr_znode}/security.json ') +
security_content,
user=params.logsearch_solr_user)
elif name == 'client':
solr_cloud_util.setup_solr_client(params.config)
if params.security_enabled:
File(format("{solr_client_dir}/logsearch_solr_client_jaas.conf"),
content=Template("logsearch_solr_jaas.conf.j2"),
owner=params.logsearch_solr_user)
else:
raise Fail('Nor client or server were selected to install.')
def metadata(type='server'):
import params
# Needed by both Server and Client
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
if type == "server":
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(format('{conf_dir}/solr'),
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
File(format("{expanded_war_dir}/atlas.war"),
content=StaticFile(
format('{metadata_home}/server/webapp/atlas.war')))
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_log4j_content))
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0644,
content=InlineTemplate(params.metadata_env_content))
if not is_empty(params.atlas_admin_username) and not is_empty(
params.atlas_admin_password):
psswd_output = hashlib.sha256(
params.atlas_admin_password).hexdigest()
ModifyPropertiesFile(
format("{conf_dir}/users-credentials.properties"),
properties={
format('{atlas_admin_username}'):
format('ROLE_ADMIN::{psswd_output}')
},
owner=params.metadata_user)
files_to_chown = [
format("{conf_dir}/atlas-simple-authz-policy.json"),
format("{conf_dir}/users-credentials.properties")
]
for file in files_to_chown:
if os.path.exists(file):
Execute(
('chown', format('{metadata_user}:{user_group}'), file),
sudo=True)
Execute(('chmod', '640', file), sudo=True)
if params.metadata_solrconfig_content:
File(format("{conf_dir}/solr/solrconfig.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_solrconfig_content))
generate_logfeeder_input_config(
'atlas',
Template("input.config-atlas.json.j2", extra_imports=[default]))
# Needed by both Server and Client
PropertiesFile(format('{conf_dir}/{conf_file}'),
properties=params.application_properties,
mode=0600,
owner=params.metadata_user,
group=params.user_group)
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)
if type == 'server' and params.search_backend_solr and params.has_infra_solr:
solr_cloud_util.setup_solr_client(params.config)
check_znode()
jaasFile = params.atlas_jaas_file if params.security_enabled else None
upload_conf_set('atlas_configs', jaasFile)
if params.security_enabled: # update permissions before creating the collections
solr_cloud_util.add_solr_roles(
params.config,
roles=[
params.infra_solr_role_atlas,
params.infra_solr_role_ranger_audit,
params.infra_solr_role_dev
],
new_service_principals=[params.atlas_jaas_principal])
create_collection('vertex_index', 'atlas_configs', jaasFile)
create_collection('edge_index', 'atlas_configs', jaasFile)
create_collection('fulltext_index', 'atlas_configs', jaasFile)
if params.security_enabled:
secure_znode(format('{infra_solr_znode}/configs/atlas_configs'),
jaasFile)
secure_znode(format('{infra_solr_znode}/collections/vertex_index'),
jaasFile)
secure_znode(format('{infra_solr_znode}/collections/edge_index'),
jaasFile)
secure_znode(
format('{infra_solr_znode}/collections/fulltext_index'),
jaasFile)
File(params.atlas_hbase_setup,
group=params.user_group,
owner=params.hbase_user,
content=Template("atlas_hbase_setup.rb.j2"))
is_atlas_upgrade_support = True
if is_atlas_upgrade_support and params.security_enabled:
File(params.atlas_kafka_setup,
group=params.user_group,
owner=params.kafka_user,
content=Template("atlas_kafka_acl.sh.j2"))
# files required only in case if kafka broker is not present on the host as configured component
if not params.host_with_kafka:
File(format("{kafka_conf_dir}/kafka-env.sh"),
owner=params.kafka_user,
content=InlineTemplate(params.kafka_env_sh_template))
File(format("{kafka_conf_dir}/kafka_jaas.conf"),
group=params.user_group,
owner=params.kafka_user,
content=Template("kafka_jaas.conf.j2"))
if params.stack_supports_atlas_hdfs_site_on_namenode_ha and len(
params.namenode_host) > 1:
XmlConfig(
"hdfs-site.xml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configurationAttributes']
['hdfs-site'],
owner=params.metadata_user,
group=params.user_group,
mode=0644)
else:
File(format('{conf_dir}/hdfs-site.xml'), action="delete")
if params.stack_supports_atlas_core_site and params.has_namenode:
XmlConfig(
"core-site.xml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configurationAttributes']
['core-site'],
owner=params.metadata_user,
group=params.user_group,
mode=0644,
xml_include_file=params.mount_table_xml_inclusion_file_full_path)
if params.mount_table_content:
File(params.mount_table_xml_inclusion_file_full_path,
owner=params.metadata_user,
group=params.user_group,
content=params.mount_table_content,
mode=0644)
Directory(
format('{metadata_home}/'),
owner=params.metadata_user,
group=params.user_group,
recursive_ownership=True,
)