def observer_namenode(action=None, format=False):
if action == "configure":
import params
for fs_checkpoint_dir in params.fs_checkpoint_dirs:
Directory(fs_checkpoint_dir,
create_parents=True,
cd_access="a",
mode=0755,
owner=params.hdfs_user,
group=params.user_group)
File(params.exclude_file_path,
content=Template("exclude_hosts_list.j2"),
owner=params.hdfs_user,
group=params.user_group)
if params.hdfs_include_file:
File(params.include_file_path,
content=Template("include_hosts_list.j2"),
owner=params.hdfs_user,
group=params.user_group)
generate_logfeeder_input_config(
'hdfs',
Template("input.config-hdfs.json.j2", extra_imports=[default]))
elif action == "start" or action == "stop":
import params
service(action=action,
name="observernamenode",
user=params.hdfs_user,
create_pid_dir=True,
create_log_dir=True)
elif action == "status":
import status_params
check_process_status(status_params.snamenode_pid_file)
def datanode(action=None):
if action == "configure":
import params
Directory(params.dfs_domain_socket_dir,
create_parents=True,
mode=0751,
owner=params.hdfs_user,
group=params.user_group)
# handle_mounted_dirs ensures that we don't create dfs data dirs which are temporary unavailable (unmounted), and intended to reside on a different mount.
data_dir_to_mount_file_content = handle_mounted_dirs(
create_dirs, params.dfs_data_dirs, params.data_dir_mount_file,
params)
# create a history file used by handle_mounted_dirs
File(params.data_dir_mount_file,
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
content=data_dir_to_mount_file_content)
generate_logfeeder_input_config(
'hdfs',
Template("input.config-hdfs.json.j2", extra_imports=[default]))
elif action == "start" or action == "stop":
import params
service(action=action,
name="datanode",
user=params.hdfs_user,
create_pid_dir=True,
create_log_dir=True)
elif action == "status":
import params
PidFiles(params.possible_datanode_pid_files).check_status()
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
self.configure(env)
generate_logfeeder_input_config(
'ambari-infra',
Template("input.config-ambari-infra.json.j2",
extra_imports=[default]))
setup_solr_znode_env()
start_cmd = format('{solr_bindir}/solr start -cloud -noprompt -s {infra_solr_datadir} -Dsolr.kerberos.name.rules=\'{infra_solr_kerberos_name_rules}\' 2>&1') \
if params.security_enabled else format('{solr_bindir}/solr start -cloud -noprompt -s {infra_solr_datadir} 2>&1')
check_process = format(
"{sudo} test -f {infra_solr_pidfile} && {sudo} pgrep -F {infra_solr_pidfile}"
)
piped_start_cmd = format('{start_cmd} | tee {infra_solr_log}'
) + '; (exit "${PIPESTATUS[0]}")'
Execute(piped_start_cmd,
environment={
'SOLR_INCLUDE':
format('{infra_solr_conf}/infra-solr-env.sh')
},
user=params.infra_solr_user,
not_if=check_process,
logoutput=True)
def configure_static(env):
import params
env.set_params(params)
generate_logfeeder_input_config('hdfs', Template("input.config-hdfs.json.j2", extra_imports=[default]))
hdfs("zkfc_slave")
utils.set_up_zkfc_security(params)
pass
def configure(self, env):
import params
env.set_params(params)
Directory(
[params.log_dirs_list, '/etc/kafka', '/var/log/kafka', '/var/log/confluent/kafka'],
owner='cp-kafka',
group='confluent',
mode=0775,
create_parents=True)
Directory([params.control_center_dir, params.kafka_log_dir],
owner='cp-kafka',
group='confluent',
mode=0777,
create_parents=True)
File(
'/etc/kafka/server.properties',
owner='cp-kafka',
group='confluent',
mode=0644,
content=InlineTemplate(params.kafka_content))
File(
'/etc/kafka/log4j.properties',
owner='cp-kafka',
group='confluent',
mode=0644,
content=log4j_content)
File(
'/etc/sysconfig/kafka',
owner='cp-kafka',
group='confluent',
mode=0644,
content=InlineTemplate(params.kafka_env_content))
generate_logfeeder_input_config(
'kafka',
Template("input.config-kafka.json.j2", extra_imports=[default]))
def configure(self, env):
import params
import status_params
env.set_params(params)
env.set_params(status_params)
# create the log, pid, conf dirs if not already present
Directory([
status_params.nifi_pid_dir, params.nifi_node_log_dir,
params.nifi_config_dir
],
owner=params.nifi_user,
group=params.nifi_group,
create_parents=True)
ca_json = os.path.join(params.nifi_config_dir,
'nifi-certificate-authority.json')
ca_dict = nifi_toolkit_util_common.load(ca_json)
nifi_toolkit_util_common.overlay(ca_dict, params.nifi_ca_config)
nifi_toolkit_util_common.dump(ca_json, ca_dict, params.nifi_user,
params.nifi_group)
generate_logfeeder_input_config(
'nifi',
Template("input.config-nifi.json.j2", extra_imports=[default]))
Directory([params.nifi_config_dir],
owner=params.nifi_user,
group=params.nifi_group,
create_parents=True,
recursive_ownership=True)
def flink(name=None):
import params
import os
Directory(params.log_dir,
owner=params.flink_user,
group=params.user_group,
mode=0777,
create_parents = True,
cd_access="a",
)
Directory([params.pid_dir],
owner=params.flink_user,
group=params.user_group,
create_parents = True,
cd_access="a",
mode=0755,
)
configurations = params.config['configurations']['flink-site']
File(format("{conf_dir}/flink-conf.yaml"),
content=yaml_config_template(configurations),
owner=params.flink_user,
group=params.user_group
)
generate_logfeeder_input_config('flink', Template("input.config-flink.json.j2", extra_imports=[default]))
if params.has_metric_collector:
File(format("{conf_dir}/flink-metrics2.properties"),
owner=params.flink_user,
group=params.user_group,
content=Template("flink-metrics2.properties.j2")
)
if params.security_enabled:
TemplateConfig(format("{conf_dir}/flink_jaas.conf"),
owner=params.flink_user,
mode=0644
)
if params.stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version_formatted):
TemplateConfig(format("{conf_dir}/client_jaas.conf"),
owner=params.flink_user,
mode=0644
)
else:
File(
format("{conf_dir}/flink_jaas.conf"),
action="delete"
)
File(
format("{conf_dir}/client_jaas.conf"),
action="delete"
)
def setup_historyserver():
import params
if params.yarn_log_aggregation_enabled:
params.HdfsResource(params.yarn_nm_app_log_dir,
action="create_on_execute",
type="directory",
owner=params.yarn_user,
group=params.user_group,
mode=01777,
recursive_chmod=True)
# create the /tmp folder with proper permissions if it doesn't exist yet
if params.entity_file_history_directory.startswith('/tmp'):
params.HdfsResource(
params.hdfs_tmp_dir,
action="create_on_execute",
type="directory",
owner=params.hdfs_user,
mode=0777,
)
params.HdfsResource(params.entity_file_history_directory,
action="create_on_execute",
type="directory",
owner=params.yarn_user,
group=params.user_group)
params.HdfsResource("/mapred",
type="directory",
action="create_on_execute",
owner=params.mapred_user)
params.HdfsResource("/mapred/system",
type="directory",
action="create_on_execute",
owner=params.hdfs_user)
params.HdfsResource(params.mapreduce_jobhistory_done_dir,
type="directory",
action="create_on_execute",
owner=params.mapred_user,
group=params.user_group,
change_permissions_for_parents=True,
mode=0777)
params.HdfsResource(None, action="execute")
Directory(
params.jhs_leveldb_state_store_dir,
owner=params.mapred_user,
group=params.user_group,
create_parents=True,
cd_access="a",
recursive_ownership=True,
)
generate_logfeeder_input_config(
'mapreduce2',
Template("input.config-mapreduce2.json.j2", extra_imports=[default]))
def configure(self, env):
import params
Directory(params.jn_edits_dirs,
create_parents = True,
cd_access="a",
owner=params.hdfs_user,
group=params.user_group
)
env.set_params(params)
generate_logfeeder_input_config('hdfs', Template("input.config-hdfs.json.j2", extra_imports=[default]))
hdfs()
pass
def setup_metastore():
import params
if params.hive_metastore_site_supported:
hivemetastore_site_config = get_config("hivemetastore-site")
if hivemetastore_site_config:
XmlConfig("hivemetastore-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']
['hivemetastore-site'],
configuration_attributes=params.
config['configurationAttributes']['hivemetastore-site'],
owner=params.hive_user,
group=params.user_group,
mode=0600)
File(os.path.join(params.hive_server_conf_dir,
"hadoop-metrics2-hivemetastore.properties"),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-hivemetastore.properties.j2"),
mode=0600)
File(params.start_metastore_path,
mode=0755,
content=StaticFile('startMetastore.sh'))
if params.hive_repl_cmrootdir is not None and params.hive_repl_cmrootdir.strip(
) != "":
params.HdfsResource(params.hive_repl_cmrootdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=01777)
if params.hive_repl_rootdir is not None and params.hive_repl_rootdir.strip(
) != "":
params.HdfsResource(params.hive_repl_rootdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=0700)
params.HdfsResource(None, action="execute")
generate_logfeeder_input_config(
'hive', Template("input.config-hive.json.j2", extra_imports=[default]))
def registry(env, upgrade_type=None):
import params
ensure_base_directories()
#Logger.info(format("Effective stack version: {effective_version}"))
File(format("{conf_dir}/registry-env.sh"),
owner=params.registry_user,
content=InlineTemplate(params.registry_env_sh_template))
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
Directory(
[params.jar_storage],
owner=params.registry_user,
group=params.user_group,
create_parents=True,
cd_access="a",
mode=0755,
)
File(os.path.join(params.limits_conf_dir, 'registry.conf'),
owner='root',
group='root',
mode=0644,
content=Template("registry.conf.j2"))
File(format("{conf_dir}/registry.yaml"),
content=Template("registry.yaml.j2"),
owner=params.registry_user,
group=params.user_group,
mode=0600)
generate_logfeeder_input_config(
'registry',
Template("input.config-registry.json.j2", extra_imports=[default]))
if not os.path.islink(params.registry_managed_log_dir):
Link(params.registry_managed_log_dir, to=params.registry_log_dir)
download_database_connector_if_needed()
def configure(self, env, isInstall=False, is_starting = False):
import params
import status_params
env.set_params(params)
env.set_params(status_params)
#create the log, pid, conf dirs if not already present
nifi_dirs = [status_params.nifi_pid_dir, params.nifi_internal_dir, params.nifi_database_dir, params.nifi_flowfile_repo_dir, params.nifi_provenance_repo_dir_default, params.nifi_config_dir, params.nifi_flow_config_dir, params.nifi_state_dir, params.lib_dir]
nifi_dirs.extend(params.nifi_content_repo_dirs)
Directory(nifi_dirs, owner=params.nifi_user, group=params.nifi_group, create_parents=True, recursive_ownership=True, cd_access='a')
# On some OS this folder may not exist, so we will create it before pushing files there
Directory(params.limits_conf_dir,
create_parents = True,
owner='root',
group='root'
)
File(os.path.join(params.limits_conf_dir, 'nifi.conf'),
owner='root',
group='root',
mode=0644,
content=Template("nifi.conf.j2")
)
#determine whether or not a cluster already exists based on zookeeper entries and determine if this is the first start of this node
#if so authorizations and flow file will not be writen
if not sudo.path_isfile(params.nifi_flow_config_dir+'/flow.xml.gz') and nifi_toolkit_util_common.existing_cluster(params):
params.is_additional_node = True
#write configurations
self.write_configurations(params, is_starting)
generate_logfeeder_input_config('nifi', Template("input.config-nifi.json.j2", extra_imports=[default]))
# if this is not an additional node being added to an existing cluster write out flow.xml.gz to internal dir only if AMS installed (must be writable by Nifi)
# and only during first install. It is used to automate setup of Ambari metrics reporting task in Nifi
if not params.is_additional_node:
if params.metrics_collector_host and params.nifi_ambari_reporting_enabled and not sudo.path_isfile(params.nifi_flow_config_dir+'/flow.xml.gz'):
Execute('echo "First time setup so generating flow.xml.gz" >> ' + params.nifi_node_log_file, user=params.nifi_user)
flow_content=InlineTemplate(params.nifi_flow_content)
File(format("{params.nifi_flow_config_dir}/flow.xml"), content=flow_content, owner=params.nifi_user, group=params.nifi_group, mode=0600)
Execute(format("cd {params.nifi_flow_config_dir}; gzip flow.xml;"), user=params.nifi_user)
def nfsgateway(action=None, format=False):
import params
if action== "start":
prepare_rpcbind()
if action == "configure":
Directory(params.nfs_file_dump_dir,
owner = params.hdfs_user,
group = params.user_group,
)
generate_logfeeder_input_config('hdfs', Template("input.config-hdfs.json.j2", extra_imports=[default]))
elif action == "start" or action == "stop":
service(
action=action,
name="nfs3",
user=params.root_user,
create_pid_dir=True,
create_log_dir=True
)
def registry(env, upgrade_type=None):
import params
ensure_base_directories()
File(format("{conf_dir}/registry-env.sh"),
owner=params.registry_user,
content=InlineTemplate(params.registry_env_sh_template))
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
Directory(
[params.jar_storage],
owner=params.registry_user,
group=params.user_group,
create_parents=True,
cd_access="a",
mode=0755,
)
File(os.path.join(params.limits_conf_dir, 'registry.conf'),
owner='root',
group='root',
mode=0644,
content=Template("registry.conf.j2"))
File(format("{conf_dir}/registry.yaml"),
content=Template("registry.yaml.j2"),
owner=params.registry_user,
group=params.user_group,
mode=0600)
generate_logfeeder_input_config(
'registry',
Template("input.config-registry.json.j2", extra_imports=[default]))
if not os.path.islink(params.registry_managed_log_dir):
Link(params.registry_managed_log_dir, to=params.registry_log_dir)
def setup_livy(env, type, upgrade_type=None, action=None):
import params
Directory([params.livy2_pid_dir, params.livy2_log_dir],
owner=params.livy2_user,
group=params.user_group,
mode=0775,
cd_access='a',
create_parents=True)
if type == 'server' and action == 'config':
params.HdfsResource(params.livy2_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.livy2_user,
mode=0775)
params.HdfsResource(None, action="execute")
params.HdfsResource(params.livy2_recovery_dir,
type="directory",
action="create_on_execute",
owner=params.livy2_user,
mode=0700)
params.HdfsResource(None, action="execute")
generate_logfeeder_input_config(
'spark2',
Template("input.config-spark2.json.j2", extra_imports=[default]))
# create livy-env.sh in etc/conf dir
File(
os.path.join(params.livy2_conf, 'livy-env.sh'),
owner=params.livy2_user,
group=params.livy2_group,
content=InlineTemplate(params.livy2_env_sh),
mode=0644,
)
# create livy.conf in etc/conf dir
PropertiesFile(
format("{livy2_conf}/livy.conf"),
properties=params.config['configurations']['livy2-conf'],
key_value_delimiter=" ",
owner=params.livy2_user,
group=params.livy2_group,
)
# create log4j.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'log4j.properties'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_log4j_properties,
mode=0644,
)
# create spark-blacklist.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'spark-blacklist.conf'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_spark_blacklist_properties,
mode=0644,
)
Directory(
params.livy2_logs_dir,
owner=params.livy2_user,
group=params.livy2_group,
mode=0755,
)
def kms(upgrade_type=None):
import params
if params.has_ranger_admin:
Directory(params.kms_conf_dir,
owner=params.kms_user,
group=params.kms_group,
create_parents=True)
Directory("/etc/security/serverKeys",
create_parents=True,
cd_access="a")
Directory("/etc/ranger/kms", create_parents=True, cd_access="a")
copy_jdbc_connector(params.kms_home)
File(
format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}/{check_db_connection_jar_name}")),
mode=0644,
)
cp = format("{check_db_connection_jar}")
if params.db_flavor.lower() == 'sqla':
cp = cp + os.pathsep + format(
"{kms_home}/ews/webapp/lib/sajdbc4.jar")
else:
path_to_jdbc = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")
if not os.path.isfile(path_to_jdbc):
path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + \
params.default_connectors_map[
params.db_flavor.lower()] if params.db_flavor.lower() in params.default_connectors_map else None
if not os.path.isfile(path_to_jdbc):
path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + "*"
error_message = "Error! Sorry, but we can't find jdbc driver with default name " + \
params.default_connectors_map[params.db_flavor] + \
" in ranger kms lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'"
Logger.error(error_message)
cp = cp + os.pathsep + path_to_jdbc
db_connection_check_command = format(
"{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_kms_jdbc_connection_url}' {db_user} {db_password!p} {ranger_kms_jdbc_driver}"
)
env_dict = {}
if params.db_flavor.lower() == 'sqla':
env_dict = {'LD_LIBRARY_PATH': params.ld_library_path}
Execute(db_connection_check_command,
path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
tries=5,
try_sleep=10,
environment=env_dict)
if params.xa_audit_db_is_enabled and params.driver_source is not None and not params.driver_source.endswith(
"/None"):
if params.xa_previous_jdbc_jar and os.path.isfile(
params.xa_previous_jdbc_jar):
File(params.xa_previous_jdbc_jar, action='delete')
File(params.downloaded_connector_path,
content=DownloadSource(params.driver_source),
mode=0644)
Execute(('cp', '--remove-destination',
params.downloaded_connector_path, params.driver_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_target, mode=0644)
Directory(os.path.join(params.kms_home, 'ews', 'webapp', 'WEB-INF',
'classes', 'lib'),
mode=0755,
owner=params.kms_user,
group=params.kms_group)
Execute(('cp', format('{kms_home}/ranger-kms-initd'),
'/etc/init.d/ranger-kms'),
not_if=format('ls /etc/init.d/ranger-kms'),
only_if=format('ls {kms_home}/ranger-kms-initd'),
sudo=True)
File('/etc/init.d/ranger-kms', mode=0755)
Directory(
format('{kms_home}/'),
owner=params.kms_user,
group=params.kms_group,
recursive_ownership=True,
)
Directory(params.ranger_kms_pid_dir,
mode=0755,
owner=params.kms_user,
group=params.user_group,
cd_access="a",
create_parents=True)
Directory(params.kms_log_dir,
owner=params.kms_user,
group=params.kms_group,
cd_access='a',
create_parents=True,
mode=0755)
generate_logfeeder_input_config(
'ranger-kms',
Template("input.config-ranger-kms.json.j2",
extra_imports=[default]))
File(format('{kms_conf_dir}/ranger-kms-env.sh'),
content=InlineTemplate(params.kms_env_content),
owner=params.kms_user,
group=params.kms_group,
mode=0755)
Execute(('ln', '-sf', format('{kms_home}/ranger-kms'),
'/usr/bin/ranger-kms'),
not_if=format('ls /usr/bin/ranger-kms'),
only_if=format('ls {kms_home}/ranger-kms'),
sudo=True)
File('/usr/bin/ranger-kms', mode=0755)
Execute(('ln', '-sf', format('{kms_home}/ranger-kms'),
'/usr/bin/ranger-kms-services.sh'),
not_if=format('ls /usr/bin/ranger-kms-services.sh'),
only_if=format('ls {kms_home}/ranger-kms'),
sudo=True)
File('/usr/bin/ranger-kms-services.sh', mode=0755)
Execute(('ln', '-sf', format('{kms_home}/ranger-kms-initd'),
format('{kms_home}/ranger-kms-services.sh')),
not_if=format('ls {kms_home}/ranger-kms-services.sh'),
only_if=format('ls {kms_home}/ranger-kms-initd'),
sudo=True)
File(format('{kms_home}/ranger-kms-services.sh'), mode=0755)
do_keystore_setup(params.credential_provider_path, params.jdbc_alias,
params.db_password)
do_keystore_setup(params.credential_provider_path,
params.masterkey_alias,
params.kms_master_key_password)
if params.stack_support_kms_hsm and params.enable_kms_hsm:
do_keystore_setup(params.credential_provider_path,
params.hms_partition_alias,
unicode(params.hms_partition_passwd))
if params.stack_supports_ranger_kms_ssl and params.ranger_kms_ssl_enabled:
do_keystore_setup(params.ranger_kms_cred_ssl_path,
params.ranger_kms_ssl_keystore_alias,
params.ranger_kms_ssl_passwd)
# remove plain-text password from xml configs
dbks_site_copy = {}
dbks_site_copy.update(params.config['configurations']['dbks-site'])
for prop in params.dbks_site_password_properties:
if prop in dbks_site_copy:
dbks_site_copy[prop] = "_"
XmlConfig(
"dbks-site.xml",
conf_dir=params.kms_conf_dir,
configurations=dbks_site_copy,
configuration_attributes=params.config['configurationAttributes']
['dbks-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644)
ranger_kms_site_copy = {}
ranger_kms_site_copy.update(
params.config['configurations']['ranger-kms-site'])
if params.stack_supports_ranger_kms_ssl:
# remove plain-text password from xml configs
for prop in params.ranger_kms_site_password_properties:
if prop in ranger_kms_site_copy:
ranger_kms_site_copy[prop] = "_"
XmlConfig(
"ranger-kms-site.xml",
conf_dir=params.kms_conf_dir,
configurations=ranger_kms_site_copy,
configuration_attributes=params.config['configurationAttributes']
['ranger-kms-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644)
XmlConfig(
"kms-site.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['kms-site'],
configuration_attributes=params.config['configurationAttributes']
['kms-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644)
File(os.path.join(params.kms_conf_dir, "kms-log4j.properties"),
owner=params.kms_user,
group=params.kms_group,
content=InlineTemplate(params.kms_log4j),
mode=0644)
# core-site.xml linking required by setup for HDFS encryption
XmlConfig(
"core-site.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configurationAttributes']
['core-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644,
xml_include_file=params.mount_table_xml_inclusion_file_full_path)
if params.mount_table_content:
File(params.mount_table_xml_inclusion_file_full_path,
owner=params.kms_user,
group=params.kms_group,
content=params.mount_table_content,
mode=0644)
def namenode(action=None,
hdfs_binary=None,
do_format=True,
upgrade_type=None,
upgrade_suspended=False,
env=None):
if action is None:
raise Fail('"action" parameter is required for function namenode().')
if action in ["start", "stop"] and hdfs_binary is None:
raise Fail(
'"hdfs_binary" parameter is required for function namenode().')
if action == "configure":
import params
#we need this directory to be present before any action(HA manual steps for
#additional namenode)
create_name_dirs(params.dfs_name_dir)
generate_logfeeder_input_config(
'hdfs',
Template("input.config-hdfs.json.j2", extra_imports=[default]))
# set up failover / secure zookeper ACLs, this feature is supported from HDP 2.6 ownwards
set_up_zkfc_security(params)
elif action == "start":
Logger.info("Called service {0} with upgrade_type: {1}".format(
action, str(upgrade_type)))
setup_ranger_hdfs(upgrade_type=upgrade_type)
import params
File(params.exclude_file_path,
content=Template("exclude_hosts_list.j2"),
owner=params.hdfs_user,
group=params.user_group)
if do_format and not params.hdfs_namenode_format_disabled:
format_namenode()
pass
if params.dfs_ha_enabled and \
len(params.dfs_ha_namenode_active) > 0 and \
params.hostname not in params.dfs_ha_namenode_active:
# if the current host is the standby NameNode in an HA deployment
# run the bootstrap command, to start the NameNode in standby mode
# this requires that the active NameNode is already up and running,
# so this execute should be re-tried upon failure, up to a timeout
success = bootstrap_standby_namenode(params)
if not success:
raise Fail("Could not bootstrap standby namenode")
if upgrade_type == constants.UPGRADE_TYPE_ROLLING and params.dfs_ha_enabled:
# Most likely, ZKFC is up since RU will initiate the failover command. However, if that failed, it would have tried
# to kill ZKFC manually, so we need to start it if not already running.
safe_zkfc_op(action, env)
summary = upgrade_summary.get_upgrade_summary()
is_downgrade_allowed = summary is not None and summary.is_downgrade_allowed
is_switch_bits = summary is not None and summary.is_switch_bits
options = ""
if is_switch_bits:
Logger.info(
"The {0} switches the binaries only. No options are used to restart NameNode."
.format(summary.direction))
else:
if upgrade_type == constants.UPGRADE_TYPE_ROLLING:
if params.upgrade_direction == Direction.UPGRADE:
options = "-rollingUpgrade started"
elif params.upgrade_direction == Direction.DOWNGRADE:
options = ""
elif upgrade_type == constants.UPGRADE_TYPE_NON_ROLLING:
is_previous_image_dir = is_previous_fs_image()
Logger.info(
"Previous file system image dir present is {0}".format(
str(is_previous_image_dir)))
if params.upgrade_direction == Direction.UPGRADE:
if is_downgrade_allowed:
options = "-rollingUpgrade started"
else:
# if we are HA, then -upgrade needs to be called for the active NN,
# then -bootstrapStandby on the other, followed by normal daemon
# if we are NOT HA, then -upgrade needs to be called on the lone NN
if params.dfs_ha_enabled:
name_service = get_name_service_by_hostname(
params.hdfs_site, params.hostname)
any_active = is_there_any_active_nn(name_service)
if any_active:
if not bootstrap_standby_namenode(
params, use_path=True,
run_if_present=True):
raise Fail(
"Could not bootstrap this namenode of an Express upgrade"
)
options = "" # we're bootstrapped, no other work needs to happen for the daemon
else:
options = "-upgrade" # no other are active, so this host's NN is the first
else:
options = "-upgrade" # non-HA
marker = os.path.exists(
namenode_upgrade.get_upgrade_in_progress_marker())
if options == "-upgrade" and upgrade_suspended is True and marker is True:
Logger.info(
"The NameNode is currently upgrading. No options will be passed to startup"
)
options = ""
elif params.upgrade_direction == Direction.DOWNGRADE:
options = ""
elif upgrade_type == constants.UPGRADE_TYPE_HOST_ORDERED:
# nothing special to do for HOU - should be very close to a normal restart
pass
elif upgrade_type is None and upgrade_suspended is True:
# the rollingUpgrade flag must be passed in during a suspended upgrade when starting NN
if os.path.exists(
namenode_upgrade.get_upgrade_in_progress_marker()):
if is_downgrade_allowed:
options = "-rollingUpgrade started"
else:
options = "-upgrade"
else:
Logger.info(
"The NameNode upgrade marker file {0} does not exist, yet an upgrade is currently suspended. "
"Assuming that the upgrade of NameNode has not occurred yet."
.format(
namenode_upgrade.get_upgrade_in_progress_marker()))
Logger.info("Options for start command are: {0}".format(options))
service(action="start",
name="namenode",
user=params.hdfs_user,
options=options,
create_pid_dir=True,
create_log_dir=True)
if params.security_enabled:
Execute(format(
"{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"
),
user=params.hdfs_user)
name_service = get_name_service_by_hostname(params.hdfs_site,
params.hostname)
# ___Scenario___________|_Expected safemode state__|_Wait for safemode OFF____|
# no-HA | ON -> OFF | Yes |
# HA and active | ON -> OFF | Yes |
# HA and standby | no change | No |
# RU with HA on active | ON -> OFF | Yes |
# RU with HA on standby | ON -> OFF | Yes |
# EU with HA on active | ON -> OFF | No |
# EU with HA on standby | ON -> OFF | No |
# EU non-HA | ON -> OFF | No |
# because we do things like create directories after starting NN,
# the vast majority of the time this should be True - it should only
# be False if this is HA and we are the Standby NN
ensure_safemode_off = True
# True if this is the only NameNode (non-HA) or if its the Active one in HA
is_active_namenode = True
if params.dfs_ha_enabled:
Logger.info(
"Waiting for the NameNode to broadcast whether it is Active or Standby..."
)
if is_this_namenode_active(name_service) is False:
# we are the STANDBY NN
is_active_namenode = False
# we are the STANDBY NN and this restart is not part of an upgrade
if upgrade_type is None:
ensure_safemode_off = False
# During an Express Upgrade, NameNode will not leave SafeMode until the DataNodes are started,
# so always disable the Safemode check
if upgrade_type == constants.UPGRADE_TYPE_NON_ROLLING:
ensure_safemode_off = False
# some informative logging separate from the above logic to keep things a little cleaner
if ensure_safemode_off:
Logger.info(
"Waiting for this NameNode to leave Safemode due to the following conditions: HA: {0}, isActive: {1}, upgradeType: {2}"
.format(params.dfs_ha_enabled, is_active_namenode,
upgrade_type))
else:
Logger.info(
"Skipping Safemode check due to the following conditions: HA: {0}, isActive: {1}, upgradeType: {2}"
.format(params.dfs_ha_enabled, is_active_namenode,
upgrade_type))
# wait for Safemode to end
if ensure_safemode_off:
if params.rolling_restart and params.rolling_restart_safemode_exit_timeout:
calculated_retries = int(
params.rolling_restart_safemode_exit_timeout) / 30
wait_for_safemode_off(hdfs_binary,
afterwait_sleep=30,
retries=calculated_retries,
sleep_seconds=30)
else:
wait_for_safemode_off(hdfs_binary)
# Always run this on the "Active" NN unless Safemode has been ignored
# in the case where safemode was ignored (like during an express upgrade), then
# NN will be in SafeMode and cannot have directories created
if is_active_namenode and ensure_safemode_off:
create_hdfs_directories(name_service)
create_ranger_audit_hdfs_directories()
else:
Logger.info(
"Skipping creation of HDFS directories since this is either not the Active NameNode or we did not wait for Safemode to finish."
)
elif action == "stop":
import params
service(action="stop", name="namenode", user=params.hdfs_user)
elif action == "status":
import status_params
check_process_status(status_params.namenode_pid_file)
elif action == "decommission":
decommission()
elif action == "refresh_nodes":
refresh_nodes()
def hive_interactive(name=None):
import params
MB_TO_BYTES = 1048576
# if warehouse directory is in DFS
if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(
params.default_fs).scheme:
# Create Hive Metastore Warehouse Dir
params.HdfsResource(params.hive_metastore_warehouse_external_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=01777)
params.HdfsResource(params.hive_metastore_warehouse_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=0700)
# create directories for llap package
pkg_dir = '/user/' + params.hive_user + '/.yarn'
for dir in [pkg_dir, pkg_dir + '/package', pkg_dir + '/package/LLAP']:
# hdfsresouces handles parent creation badly
params.HdfsResource(dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=01755)
if not is_empty(params.tez_hook_proto_base_directory):
params.HdfsResource(params.tez_hook_proto_base_directory,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01755)
if not is_empty(params.hive_hook_proto_base_directory):
params.HdfsResource(params.hive_hook_proto_base_directory,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
dag_meta = params.tez_hook_proto_base_directory + "dag_meta"
params.HdfsResource(dag_meta,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
dag_data = params.tez_hook_proto_base_directory + "dag_data"
params.HdfsResource(dag_data,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
app_data = params.tez_hook_proto_base_directory + "app_data"
params.HdfsResource(app_data,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
else:
Logger.info(
format(
"Not creating warehouse directory '{hive_metastore_warehouse_dir}', as the location is not in DFS."
))
# Create Hive User Dir
params.HdfsResource(params.hive_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=params.hive_hdfs_user_mode)
params.HdfsResource(None, action="execute")
# list of properties that should be excluded from the config
# this approach is a compromise against adding a dedicated config
# type for hive_server_interactive or needed config groups on a
# per component basis
exclude_list = ['hive.enforce.bucketing', 'hive.enforce.sorting']
# List of configs to be excluded from hive2 client, but present in Hive2 server.
exclude_list_for_hive2_client = [
'javax.jdo.option.ConnectionPassword',
'hadoop.security.credential.provider.path'
]
Logger.info("Directories to fill with configs: %s" %
str(params.hive_conf_dirs_list))
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
'''
As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml,
we need to merge hive/hive-site.xml and hive2/hive-site.xml and store it in hive2/hive-site.xml.
'''
merged_hive_interactive_site = {}
merged_hive_interactive_site.update(params.hive_site_config)
merged_hive_interactive_site.update(
params.config['configurations']['hive-interactive-site'])
for item in exclude_list:
if item in merged_hive_interactive_site.keys():
del merged_hive_interactive_site[item]
merged_hive_interactive_site[
'hive.llap.daemon.vcpus.per.instance'] = format(
merged_hive_interactive_site['hive.llap.daemon.vcpus.per.instance']
)
merged_hive_interactive_site[
'hive.server2.active.passive.ha.enable'] = str(
params.hive_server_interactive_ha).lower()
'''
Config 'hive.llap.io.memory.size' calculated value in stack_advisor is in MB as of now. We need to
convert it to bytes before we write it down to config file.
'''
if 'hive.llap.io.memory.size' in merged_hive_interactive_site.keys():
hive_llap_io_mem_size_in_mb = merged_hive_interactive_site.get(
"hive.llap.io.memory.size")
hive_llap_io_mem_size_in_bytes = long(
hive_llap_io_mem_size_in_mb) * MB_TO_BYTES
merged_hive_interactive_site[
'hive.llap.io.memory.size'] = hive_llap_io_mem_size_in_bytes
Logger.info(
"Converted 'hive.llap.io.memory.size' value from '{0} MB' to '{1} Bytes' before writing "
"it to config file.".format(hive_llap_io_mem_size_in_mb,
hive_llap_io_mem_size_in_bytes))
'''
Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook',
which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed.
'''
# Generate atlas-application.properties.xml file
if params.enable_atlas_hook and params.stack_supports_atlas_hook_for_hive_interactive:
Logger.info("Setup for Atlas Hive2 Hook started.")
atlas_hook_filepath = os.path.join(
params.hive_server_interactive_conf_dir,
params.atlas_hook_filename)
setup_atlas_hook(SERVICE.HIVE,
params.hive_atlas_application_properties,
atlas_hook_filepath, params.hive_user,
params.user_group)
Logger.info("Setup for Atlas Hive2 Hook done.")
else:
# Required for HDP 2.5 stacks
Logger.info(
"Skipping setup for Atlas Hook, as it is disabled/ not supported.")
remove_atlas_hook_if_exists(merged_hive_interactive_site)
'''
As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml,
we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml.
'''
merged_tez_interactive_site = {}
if 'tez-site' in params.config['configurations']:
merged_tez_interactive_site.update(
params.config['configurations']['tez-site'])
Logger.info(
"Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'."
)
else:
Logger.error(
"Tez's 'tez-site' couldn't be retrieved from passed-in configurations."
)
merged_tez_interactive_site.update(
params.config['configurations']['tez-interactive-site'])
XmlConfig("tez-site.xml",
conf_dir=params.tez_interactive_conf_dir,
configurations=merged_tez_interactive_site,
configuration_attributes=params.config['configurationAttributes']
['tez-interactive-site'],
owner=params.tez_interactive_user,
group=params.user_group,
mode=0664)
'''
Merge properties from hiveserver2-interactive-site into hiveserver2-site
'''
merged_hiveserver2_interactive_site = {}
if 'hiveserver2-site' in params.config['configurations']:
merged_hiveserver2_interactive_site.update(
params.config['configurations']['hiveserver2-site'])
Logger.info(
"Retrieved 'hiveserver2-site' for merging with 'hiveserver2-interactive-site'."
)
else:
Logger.error(
"'hiveserver2-site' couldn't be retrieved from passed-in configurations."
)
merged_hiveserver2_interactive_site.update(
params.config['configurations']['hiveserver2-interactive-site'])
# Create config files under hive_server_interactive_conf_dir:
# hive-site.xml
# hive-env.sh
# llap-daemon-log4j2.properties
# llap-cli-log4j2.properties
# hive-log4j2.properties
# hive-exec-log4j2.properties
# beeline-log4j2.properties
hive_server_interactive_conf_dir = params.hive_server_interactive_conf_dir
mode_identified = 0600
merged_hive_interactive_site = update_credential_provider_path(
merged_hive_interactive_site, 'hive-site',
os.path.join(conf_dir, 'hive-site.jceks'), params.hive_user,
params.user_group)
XmlConfig("hive-site.xml",
conf_dir=hive_server_interactive_conf_dir,
configurations=merged_hive_interactive_site,
configuration_attributes=params.config['configurationAttributes']
['hive-interactive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
XmlConfig("hiveserver2-site.xml",
conf_dir=hive_server_interactive_conf_dir,
configurations=merged_hiveserver2_interactive_site,
configuration_attributes=params.config['configurationAttributes']
['hiveserver2-interactive-site'],
owner=params.hive_user,
group=params.user_group,
mode=mode_identified)
File(format("{hive_server_interactive_conf_dir}/hive-env.sh"),
owner=params.hive_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.hive_interactive_env_sh_template))
llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.llap_daemon_log4j))
llap_cli_log4j2_filename = 'llap-cli-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.llap_cli_log4j2))
hive_log4j2_filename = 'hive-log4j2.properties'
File(format("{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.hive_log4j2))
hive_exec_log4j2_filename = 'hive-exec-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.hive_exec_log4j2))
beeline_log4j2_filename = 'beeline-log4j2.properties'
File(
format("{hive_server_interactive_conf_dir}/{beeline_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.beeline_log4j2))
XmlConfig("beeline-site.xml",
conf_dir=conf_dir,
configurations=params.beeline_site_config,
owner=params.hive_user,
group=params.user_group,
mode=mode_identified)
File(os.path.join(hive_server_interactive_conf_dir,
"hadoop-metrics2-hiveserver2.properties"),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=Template("hadoop-metrics2-hiveserver2.properties.j2"))
File(format(
"{hive_server_interactive_conf_dir}/hadoop-metrics2-llapdaemon.properties"
),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=Template("hadoop-metrics2-llapdaemon.j2"))
File(format(
"{hive_server_interactive_conf_dir}/hadoop-metrics2-llaptaskscheduler.properties"
),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=Template("hadoop-metrics2-llaptaskscheduler.j2"))
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'hive.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hive.conf.j2"))
if not os.path.exists(params.target_hive_interactive):
jdbc_connector(params.target_hive_interactive,
params.hive_intaractive_previous_jdbc_jar)
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}/{check_db_connection_jar_name}")),
mode=0644)
File(params.start_hiveserver2_interactive_path,
mode=0755,
content=Template(format('{start_hiveserver2_interactive_script}')))
Directory(params.hive_pid_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_log_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_interactive_var_lib,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
generate_logfeeder_input_config(
'hive', Template("input.config-hive.json.j2", extra_imports=[default]))
def setup_usersync(upgrade_type=None):
import params
usersync_home = params.usersync_home
ranger_home = params.ranger_home
ranger_ugsync_conf = params.ranger_ugsync_conf
if not is_empty(
params.ranger_usersync_ldap_ldapbindpassword
) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder':
password_validation(params.ranger_usersync_ldap_ldapbindpassword)
Directory(params.ranger_pid_dir,
mode=0755,
owner=params.unix_user,
group=params.user_group,
cd_access="a",
create_parents=True)
Directory(params.usersync_log_dir,
owner=params.unix_user,
group=params.unix_group,
cd_access='a',
create_parents=True,
mode=0755,
recursive_ownership=True)
Directory(format("{ranger_ugsync_conf}/"), owner=params.unix_user)
generate_logfeeder_input_config(
'ranger',
Template("input.config-ranger.json.j2", extra_imports=[default]))
if upgrade_type is not None:
src_file = format(
'{usersync_home}/conf.dist/ranger-ugsync-default.xml')
dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
if params.stack_supports_ranger_log4j:
File(format('{usersync_home}/conf/log4j.properties'),
owner=params.unix_user,
group=params.unix_group,
content=InlineTemplate(params.usersync_log4j),
mode=0644)
elif upgrade_type is not None and not params.stack_supports_ranger_log4j:
src_file = format('{usersync_home}/conf.dist/log4j.xml')
dst_file = format('{usersync_home}/conf/log4j.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
# remove plain-text password from xml configs
ranger_ugsync_site_copy = {}
ranger_ugsync_site_copy.update(
params.config['configurations']['ranger-ugsync-site'])
for prop in params.ranger_usersync_password_properties:
if prop in ranger_ugsync_site_copy:
ranger_ugsync_site_copy[prop] = "_"
XmlConfig("ranger-ugsync-site.xml",
conf_dir=ranger_ugsync_conf,
configurations=ranger_ugsync_site_copy,
configuration_attributes=params.config['configurationAttributes']
['ranger-ugsync-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
if os.path.isfile(params.ranger_ugsync_default_file):
File(params.ranger_ugsync_default_file,
owner=params.unix_user,
group=params.unix_group)
if os.path.isfile(params.usgsync_log4j_file):
File(params.usgsync_log4j_file,
owner=params.unix_user,
group=params.unix_group)
if os.path.isfile(params.cred_validator_file):
File(params.cred_validator_file, group=params.unix_group, mode=04555)
ranger_credential_helper(params.ugsync_cred_lib,
'usersync.ssl.key.password',
params.ranger_usersync_keystore_password,
params.ugsync_jceks_path)
if not is_empty(
params.ranger_usersync_ldap_ldapbindpassword
) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder':
ranger_credential_helper(params.ugsync_cred_lib,
'ranger.usersync.ldap.bindalias',
params.ranger_usersync_ldap_ldapbindpassword,
params.ugsync_jceks_path)
ranger_credential_helper(params.ugsync_cred_lib,
'usersync.ssl.truststore.password',
params.ranger_usersync_truststore_password,
params.ugsync_jceks_path)
File(params.ugsync_jceks_path,
owner=params.unix_user,
group=params.unix_group,
only_if=format("test -e {ugsync_jceks_path}"),
mode=0640)
update_dot_jceks_crc_ownership(
credential_provider_path=params.ugsync_jceks_path,
user=params.unix_user,
group=params.unix_group)
File(
params.usersync_services_file,
mode=0755,
)
if not os.path.isfile(params.ranger_usersync_keystore_file):
cmd = format(
"{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'"
)
Execute(cmd, logoutput=True, user=params.unix_user)
File(params.ranger_usersync_keystore_file,
owner=params.unix_user,
group=params.unix_group,
mode=0640)
create_core_site_xml(ranger_ugsync_conf)
File(format("{ranger_ugsync_conf}/ranger-usersync-env.sh"),
content=InlineTemplate(params.ranger_env_content),
owner=params.unix_user,
group=params.unix_group,
mode=0755)
def yarn(name=None, config_dir=None):
"""
:param name: Component name, apptimelinereader, apptimelineserver, nodemanager, resourcemanager, or None (defaults for client)
:param config_dir: Which config directory to write configs to, which could be different during rolling upgrade.
"""
import params
install_lzo_if_needed()
if config_dir is None:
config_dir = params.hadoop_conf_dir
Directory(
[params.yarn_log_dir_prefix],
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
ignore_failures=True,
cd_access='a',
mode=0775,
)
Directory(
[params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir],
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
cd_access='a',
)
Directory(
[
params.mapred_pid_dir_prefix, params.mapred_pid_dir,
params.mapred_log_dir_prefix, params.mapred_log_dir
],
owner=params.mapred_user,
group=params.user_group,
create_parents=True,
cd_access='a',
)
Directory(
params.yarn_hbase_conf_dir,
owner=params.yarn_hbase_user,
group=params.user_group,
create_parents=True,
cd_access='a',
)
# Some of these function calls depend on the directories above being created first.
if name == 'resourcemanager':
setup_resourcemanager()
elif name == 'nodemanager':
setup_nodemanager()
elif name == 'apptimelineserver':
setup_ats()
elif name == 'historyserver':
setup_historyserver()
elif name == 'apptimelinereader':
if not params.is_hbase_system_service_launch:
setup_atsv2_hbase_directories()
setup_atsv2_hbase_files()
generate_logfeeder_input_config(
'yarn', Template("input.config-yarn.json.j2", extra_imports=[default]))
# if there is the viewFS mount table content, create separate xml config and include in in the core-site
# else just create core-site
if params.mount_table_content:
XmlConfig(
"core-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configurationAttributes']
['core-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
xml_include_file=os.path.join(config_dir,
params.xml_inclusion_file_name))
File(os.path.join(config_dir, params.xml_inclusion_file_name),
owner=params.hdfs_user,
group=params.user_group,
content=params.mount_table_content,
mode=0644)
else:
XmlConfig(
"core-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configurationAttributes']
['core-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644)
# During RU, Core Masters and Slaves need hdfs-site.xml
# TODO, instead of specifying individual configs, which is susceptible to breaking when new configs are added,
# RU should rely on all available in <stack-root>/<version>/hadoop/conf
XmlConfig("hdfs-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configurationAttributes']
['hdfs-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644)
XmlConfig("mapred-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configurationAttributes']
['mapred-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
configs = {}
configs.update(params.config['configurations']['yarn-site'])
configs["hadoop.registry.dns.bind-port"] = params.config['configurations'][
'yarn-env']['registry.dns.bind-port']
XmlConfig("yarn-site.xml",
conf_dir=config_dir,
configurations=configs,
configuration_attributes=params.config['configurationAttributes']
['yarn-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
XmlConfig(
"capacity-scheduler.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['capacity-scheduler'],
configuration_attributes=params.config['configurationAttributes']
['capacity-scheduler'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
XmlConfig(
"hbase-site.xml",
conf_dir=params.yarn_hbase_conf_dir,
configurations=params.config['configurations']['yarn-hbase-site'],
configuration_attributes=params.config['configurationAttributes']
['yarn-hbase-site'],
owner=params.yarn_hbase_user,
group=params.user_group,
mode=0644)
XmlConfig("resource-types.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['resource-types'],
configuration_attributes=params.config['configurationAttributes']
['resource-types'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
File(format("{limits_conf_dir}/yarn.conf"),
mode=0644,
content=Template('yarn.conf.j2'))
File(format("{limits_conf_dir}/mapreduce.conf"),
mode=0644,
content=Template('mapreduce.conf.j2'))
File(os.path.join(config_dir, "yarn-env.sh"),
owner=params.yarn_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.yarn_env_sh_template))
File(format("{yarn_bin}/container-executor"),
group=params.yarn_executor_container_group,
mode=params.container_executor_mode)
File(os.path.join(config_dir, "container-executor.cfg"),
group=params.user_group,
mode=0644,
content=InlineTemplate(params.container_executor_cfg_template))
Directory(params.cgroups_dir,
group=params.user_group,
create_parents=True,
mode=0755,
cd_access="a")
File(os.path.join(config_dir, "mapred-env.sh"),
owner=params.tc_owner,
mode=0755,
content=InlineTemplate(params.mapred_env_sh_template))
if params.yarn_nodemanager_recovery_dir:
Directory(
InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
mode=0755,
cd_access='a',
)
if params.security_enabled:
File(os.path.join(params.hadoop_bin, "task-controller"),
owner="root",
group=params.mapred_tt_group,
mode=06050)
File(os.path.join(config_dir, 'taskcontroller.cfg'),
owner=params.tc_owner,
mode=params.tc_mode,
group=params.mapred_tt_group,
content=Template("taskcontroller.cfg.j2"))
File(os.path.join(config_dir, 'yarn_jaas.conf'),
owner=params.yarn_user,
group=params.user_group,
content=Template("yarn_jaas.conf.j2"))
if params.has_ats:
File(os.path.join(config_dir, 'yarn_ats_jaas.conf'),
owner=params.yarn_user,
group=params.user_group,
content=Template("yarn_ats_jaas.conf.j2"))
if params.has_registry_dns:
File(os.path.join(config_dir, 'yarn_registry_dns_jaas.conf'),
owner=params.yarn_user,
group=params.user_group,
content=Template("yarn_registry_dns_jaas.conf.j2"))
File(os.path.join(config_dir, 'yarn_nm_jaas.conf'),
owner=params.yarn_user,
group=params.user_group,
content=Template("yarn_nm_jaas.conf.j2"))
if params.has_hs:
File(os.path.join(config_dir, 'mapred_jaas.conf'),
owner=params.mapred_user,
group=params.user_group,
content=Template("mapred_jaas.conf.j2"))
else:
File(os.path.join(config_dir, 'taskcontroller.cfg'),
owner=params.tc_owner,
content=Template("taskcontroller.cfg.j2"))
XmlConfig("mapred-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configurationAttributes']
['mapred-site'],
owner=params.mapred_user,
group=params.user_group)
XmlConfig(
"capacity-scheduler.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['capacity-scheduler'],
configuration_attributes=params.config['configurationAttributes']
['capacity-scheduler'],
owner=params.hdfs_user,
group=params.user_group)
if "ssl-client" in params.config['configurations']:
XmlConfig(
"ssl-client.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configurationAttributes']
['ssl-client'],
owner=params.hdfs_user,
group=params.user_group)
Directory(
params.hadoop_conf_secure_dir,
create_parents=True,
owner='root',
group=params.user_group,
cd_access='a',
)
XmlConfig(
"ssl-client.xml",
conf_dir=params.hadoop_conf_secure_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configurationAttributes']
['ssl-client'],
owner=params.hdfs_user,
group=params.user_group)
if "ssl-server" in params.config['configurations']:
XmlConfig(
"ssl-server.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['ssl-server'],
configuration_attributes=params.config['configurationAttributes']
['ssl-server'],
owner=params.hdfs_user,
group=params.user_group)
if os.path.exists(os.path.join(config_dir, 'fair-scheduler.xml')):
File(os.path.join(config_dir, 'fair-scheduler.xml'),
owner=params.mapred_user,
group=params.user_group)
if os.path.exists(os.path.join(config_dir, 'ssl-client.xml.example')):
File(os.path.join(config_dir, 'ssl-client.xml.example'),
owner=params.mapred_user,
group=params.user_group)
if os.path.exists(os.path.join(config_dir, 'ssl-server.xml.example')):
File(os.path.join(config_dir, 'ssl-server.xml.example'),
owner=params.mapred_user,
group=params.user_group)
setup_atsv2_backend(name, config_dir)
def oozie(is_server=False, upgrade_type=None):
import params
if is_server:
params.HdfsResource(params.oozie_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.oozie_user,
mode=params.oozie_hdfs_user_mode)
params.HdfsResource(None, action="execute")
generate_logfeeder_input_config(
'oozie',
Template("input.config-oozie.json.j2", extra_imports=[default]))
Directory(params.conf_dir,
create_parents=True,
owner=params.oozie_user,
group=params.user_group)
params.oozie_site = update_credential_provider_path(
params.oozie_site,
'oozie-site',
os.path.join(params.conf_dir, 'oozie-site.jceks'),
params.oozie_user,
params.user_group,
use_local_jceks=True)
XmlConfig("oozie-site.xml",
conf_dir=params.conf_dir,
configurations=params.oozie_site,
configuration_attributes=params.config['configurationAttributes']
['oozie-site'],
owner=params.oozie_user,
group=params.user_group,
mode=0664)
File(
format("{conf_dir}/oozie-env.sh"),
owner=params.oozie_user,
content=InlineTemplate(params.oozie_env_sh_template),
group=params.user_group,
)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'oozie.conf'),
owner='root',
group='root',
mode=0644,
content=Template("oozie.conf.j2"))
if (params.log4j_props != None):
File(format("{params.conf_dir}/oozie-log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.oozie_user,
content=InlineTemplate(params.log4j_props))
elif (os.path.exists(format("{params.conf_dir}/oozie-log4j.properties"))):
File(format("{params.conf_dir}/oozie-log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.oozie_user)
if params.stack_version_formatted and check_stack_feature(
StackFeature.OOZIE_ADMIN_USER, params.stack_version_formatted):
File(format("{params.conf_dir}/adminusers.txt"),
mode=0644,
group=params.user_group,
owner=params.oozie_user,
content=Template('adminusers.txt.j2',
oozie_admin_users=params.oozie_admin_users))
else:
File(format("{params.conf_dir}/adminusers.txt"),
owner=params.oozie_user,
group=params.user_group)
if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \
params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
params.jdbc_driver_name == "org.postgresql.Driver" or \
params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver":
File(
format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}/{check_db_connection_jar_name}")),
)
pass
oozie_ownership()
if params.lzo_enabled:
install_lzo_if_needed()
Execute(
format(
'{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
)
if is_server:
oozie_server_specific(upgrade_type)
def druid(upgrade_type=None, nodeType=None):
import params
ensure_base_directories()
# Environment Variables
File(format("{params.druid_conf_dir}/druid-env.sh"),
owner=params.druid_user,
content=InlineTemplate(params.druid_env_sh_template),
mode=0700)
# common config
druid_common_config = mutable_config_dict(
params.config['configurations']['druid-common'])
# User cannot override below configs
druid_common_config['druid.host'] = params.hostname
druid_common_config[
'druid.extensions.directory'] = params.druid_extensions_dir
druid_common_config[
'druid.extensions.hadoopDependenciesDir'] = params.druid_hadoop_dependencies_dir
druid_common_config[
'druid.selectors.indexing.serviceName'] = params.config[
'configurations']['druid-overlord']['druid.service']
druid_common_config['druid.selectors.coordinator.serviceName'] = \
params.config['configurations']['druid-coordinator']['druid.service']
druid_common_config['druid.extensions.loadList'] = json.dumps(
eval(params.druid_extensions_load_list) +
eval(params.druid_security_extensions_load_list))
# delete the password and user if empty otherwiswe derby will fail.
if 'derby' == druid_common_config['druid.metadata.storage.type']:
del druid_common_config['druid.metadata.storage.connector.user']
del druid_common_config['druid.metadata.storage.connector.password']
druid_env_config = mutable_config_dict(
params.config['configurations']['druid-env'])
PropertiesFile("common.runtime.properties",
dir=params.druid_common_conf_dir,
properties=druid_common_config,
owner=params.druid_user,
group=params.user_group,
mode=0600)
Logger.info("Created common.runtime.properties")
File(format("{params.druid_common_conf_dir}/druid-log4j.xml"),
mode=0644,
owner=params.druid_user,
group=params.user_group,
content=InlineTemplate(params.log4j_props))
Logger.info("Created log4j file")
File("/etc/logrotate.d/druid",
mode=0644,
owner='root',
group='root',
content=InlineTemplate(params.logrotate_props))
Logger.info("Created log rotate file")
if params.security_enabled:
TemplateConfig(params.druid_jaas_file,
owner=params.druid_user,
group=params.user_group,
mode=0644)
# node specific configs
for node_type in [
'coordinator', 'overlord', 'historical', 'broker', 'middleManager',
'router'
]:
node_config_dir = format('{params.druid_conf_dir}/{node_type}')
node_type_lowercase = node_type.lower()
# Write runtime.properties file
node_config = mutable_config_dict(params.config['configurations'][
format('druid-{node_type_lowercase}')])
if (node_type == 'middleManager'):
# Replace correct values for stack_version and druid_jaas_file
node_config['druid.indexer.runner.javaOpts'] = format(
node_config['druid.indexer.runner.javaOpts'])
PropertiesFile("runtime.properties",
dir=node_config_dir,
properties=node_config,
owner=params.druid_user,
group=params.user_group,
mode=0600)
Logger.info(
format("Created druid-{node_type_lowercase} runtime.properties"))
# Write jvm configs
File(
format('{node_config_dir}/jvm.config'),
owner=params.druid_user,
group=params.user_group,
content=InlineTemplate(
"-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
node_heap_memory=druid_env_config[format(
'druid.{node_type_lowercase}.jvm.heap.memory')],
log4j_config_file=format(
"{params.druid_common_conf_dir}/druid-log4j.xml"),
node_direct_memory=druid_env_config[format(
'druid.{node_type_lowercase}.jvm.direct.memory')],
node_jvm_opts=format(druid_env_config[format(
'druid.{node_type_lowercase}.jvm.opts')])))
Logger.info(format("Created druid-{node_type_lowercase} jvm.config"))
# Handling hadoop Lzo jars if enable and node type is hadoop related eg Overlords and MMs
if ['middleManager', 'overlord'
].__contains__(node_type_lowercase) and params.lzo_enabled:
try:
Logger.info(
format(
"Copying hadoop lzo jars from {hadoop_lib_home} to {druid_hadoop_dependencies_dir}/hadoop-client/*/"
))
Execute(
format(
'{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {druid_hadoop_dependencies_dir}/hadoop-client/*/'
))
except Fail as ex:
Logger.info(
format(
"No Hadoop LZO found at {hadoop_lib_home}/hadoop-lzo*.jar"
))
generate_logfeeder_input_config(
'druid', Template("input.config-druid.json.j2",
extra_imports=[default]))
# All druid nodes have dependency on hdfs_client
ensure_hadoop_directories()
download_database_connector_if_needed()
# Pull all required dependencies
pulldeps()
def flume(action=None):
import params
if action == 'config':
if params.security_enabled:
File(
format("{flume_conf_dir}/flume_jaas.conf"),
owner=params.flume_user,
content=InlineTemplate(params.flume_jaas_conf_template))
Directory([params.flume_log_dir, params.flume_conf_dir, params.flume_run_dir],
owner=params.flume_user,
group=params.flume_group,
mode=0775,
cd_access="a",
create_parents=True
)
File(params.flume_conf_dir + '/flume.conf',
content=InlineTemplate(params.flume_conf_content),
owner=params.flume_user,
mode=0644)
File(
params.flume_conf_dir + '/log4j.properties',
content=Template('log4j.properties.j2'),
owner=params.flume_user,
mode=0644)
File(
params.flume_conf_dir + '/flume-env.sh',
owner=params.flume_user,
content=InlineTemplate(params.flume_env_sh_template))
if params.has_metric_collector:
File(params.flume_conf_dir + '/flume-metrics2.properties',
owner=params.flume_user,
content=Template("flume-metrics2.properties.j2"))
generate_logfeeder_input_config(
'flume',
Template("input.config-flume.json.j2", extra_imports=[default]))
elif action == 'start':
flume_base = as_user(
format(
"{flume_bin} agent --name {{0}} --conf {{1}} --conf-file {{2}} {{3}} > {flume_log_dir}/flume.out 2>&1"
),
params.flume_user,
env={'JAVA_HOME': params.java_home}) + " &"
extra_args = ''
if params.has_metric_collector:
extra_args = '-Dflume.monitoring.type=org.apache.hadoop.metrics2.sink.flume.FlumeTimelineMetricsSink ' \
'-Dflume.monitoring.node={0}:{1}'
extra_args = extra_args.format(
params.metric_collector_host,
params.metric_collector_port)
flume_cmd = flume_base.format('ingest', params.flume_conf_dir,
params.flume_conf_dir + '/flume.conf',
extra_args)
Execute(
flume_cmd,
wait_for_finish=False,
environment={'JAVA_HOME': params.java_home})
pid_cmd = as_sudo(('pgrep', '-o', '-u', params.flume_user, '-f', format('^{java_home}'))) + \
" | " + as_sudo(('tee', params.flume_pid_file)) + " && test ${PIPESTATUS[0]} -eq 0"
try:
Execute(pid_cmd, logoutput=True, tries=20, try_sleep=10)
except:
show_logs(params.flume_log_dir, params.flume_user)
elif action == 'stop':
pid = shell.checked_call(("cat", params.flume_pid_file), sudo=True)[1].strip()
Execute(("kill", "-15", pid), sudo=True)
show_logs(params.flume_log_dir, params.flume_user)
File(params.flume_pid_file, action='delete')
def metadata(type='server'):
import params
# Needed by both Server and Client
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
if type == "server":
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(format('{conf_dir}/solr'),
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
File(format("{expanded_war_dir}/atlas.war"),
content=StaticFile(
format('{metadata_home}/server/webapp/atlas.war')))
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_log4j_content))
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content))
if not is_empty(params.atlas_admin_username) and not is_empty(
params.atlas_admin_password):
psswd_output = hashlib.sha256(
params.atlas_admin_password).hexdigest()
ModifyPropertiesFile(
format("{conf_dir}/users-credentials.properties"),
properties={
format('{atlas_admin_username}'):
format('ROLE_ADMIN::{psswd_output}')
},
owner=params.metadata_user)
files_to_chown = [
format("{conf_dir}/atlas-simple-authz-policy.json"),
format("{conf_dir}/users-credentials.properties")
]
for file in files_to_chown:
if os.path.exists(file):
Execute(
('chown', format('{metadata_user}:{user_group}'), file),
sudo=True)
Execute(('chmod', '640', file), sudo=True)
if params.metadata_solrconfig_content:
File(format("{conf_dir}/solr/solrconfig.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_solrconfig_content))
generate_logfeeder_input_config(
'atlas',
Template("input.config-atlas.json.j2", extra_imports=[default]))
# Needed by both Server and Client
PropertiesFile(format('{conf_dir}/{conf_file}'),
properties=params.application_properties,
mode=0600,
owner=params.metadata_user,
group=params.user_group)
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)
if type == 'server' and params.search_backend_solr and params.has_infra_solr:
solr_cloud_util.setup_solr_client(params.config)
check_znode()
jaasFile = params.atlas_jaas_file if params.security_enabled else None
upload_conf_set('atlas_configs', jaasFile)
if params.security_enabled: # update permissions before creating the collections
solr_cloud_util.add_solr_roles(
params.config,
roles=[
params.infra_solr_role_atlas,
params.infra_solr_role_ranger_audit,
params.infra_solr_role_dev
],
new_service_principals=[params.atlas_jaas_principal])
create_collection('vertex_index', 'atlas_configs', jaasFile)
create_collection('edge_index', 'atlas_configs', jaasFile)
create_collection('fulltext_index', 'atlas_configs', jaasFile)
if params.security_enabled:
secure_znode(format('{infra_solr_znode}/configs/atlas_configs'),
jaasFile)
secure_znode(format('{infra_solr_znode}/collections/vertex_index'),
jaasFile)
secure_znode(format('{infra_solr_znode}/collections/edge_index'),
jaasFile)
secure_znode(
format('{infra_solr_znode}/collections/fulltext_index'),
jaasFile)
File(params.atlas_hbase_setup,
group=params.user_group,
owner=params.hbase_user,
content=Template("atlas_hbase_setup.rb.j2"))
is_atlas_upgrade_support = check_stack_feature(
StackFeature.ATLAS_UPGRADE_SUPPORT,
get_stack_feature_version(params.config))
if is_atlas_upgrade_support and params.security_enabled:
File(params.atlas_kafka_setup,
group=params.user_group,
owner=params.kafka_user,
content=Template("atlas_kafka_acl.sh.j2"))
# files required only in case if kafka broker is not present on the host as configured component
if not params.host_with_kafka:
File(format("{kafka_conf_dir}/kafka-env.sh"),
owner=params.kafka_user,
content=InlineTemplate(params.kafka_env_sh_template))
File(format("{kafka_conf_dir}/kafka_jaas.conf"),
group=params.user_group,
owner=params.kafka_user,
content=Template("kafka_jaas.conf.j2"))
if params.stack_supports_atlas_hdfs_site_on_namenode_ha and len(
params.namenode_host) > 1:
XmlConfig(
"hdfs-site.xml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configurationAttributes']
['hdfs-site'],
owner=params.metadata_user,
group=params.user_group,
mode=0644)
else:
File(format('{conf_dir}/hdfs-site.xml'), action="delete")
'''
Atlas requires hadoop core-site.xml to resolve users/groups synced in HadoopUGI for
authentication and authorization process. Earlier the core-site.xml was available in
Hbase conf directory which is a part of Atlas class-path, from stack 2.6 onwards,
core-site.xml is no more available in Hbase conf directory. Hence need to create
core-site.xml in Atlas conf directory.
'''
if params.stack_supports_atlas_core_site and params.has_namenode:
XmlConfig(
"core-site.xml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configurationAttributes']
['core-site'],
owner=params.metadata_user,
group=params.user_group,
mode=0644,
xml_include_file=params.mount_table_xml_inclusion_file_full_path)
if params.mount_table_content:
File(params.mount_table_xml_inclusion_file_full_path,
owner=params.metadata_user,
group=params.user_group,
content=params.mount_table_content,
mode=0644)
Directory(
format('{metadata_home}/'),
owner=params.metadata_user,
group=params.user_group,
recursive_ownership=True,
)
def setup_spark(env, type, upgrade_type = None, action = None):
import params
# ensure that matching LZO libraries are installed for Spark
lzo_utils.install_lzo_if_needed()
Directory([params.spark_pid_dir, params.spark_log_dir],
owner=params.spark_user,
group=params.user_group,
mode=0775,
create_parents = True
)
if type == 'server' and action == 'config':
params.HdfsResource(params.spark_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.spark_user,
mode=0775
)
if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(params.default_fs).scheme:
# Create Spark Warehouse Dir
params.HdfsResource(params.spark_warehouse_dir,
type="directory",
action="create_on_execute",
owner=params.spark_user,
mode=0777
)
params.HdfsResource(None, action="execute")
generate_logfeeder_input_config('spark2', Template("input.config-spark2.json.j2", extra_imports=[default]))
spark2_defaults = dict(params.config['configurations']['spark2-defaults'])
if params.security_enabled:
spark2_defaults.pop("history.server.spnego.kerberos.principal")
spark2_defaults.pop("history.server.spnego.keytab.file")
spark2_defaults['spark.history.kerberos.principal'] = spark2_defaults['spark.history.kerberos.principal'].replace('_HOST', socket.getfqdn().lower())
PropertiesFile(format("{spark_conf}/spark-defaults.conf"),
properties = spark2_defaults,
key_value_delimiter = " ",
owner=params.spark_user,
group=params.spark_group,
mode=0644
)
# create spark-env.sh in etc/conf dir
File(os.path.join(params.spark_conf, 'spark-env.sh'),
owner=params.spark_user,
group=params.spark_group,
content=InlineTemplate(params.spark_env_sh),
mode=0644,
)
#create log4j.properties in etc/conf dir
File(os.path.join(params.spark_conf, 'log4j.properties'),
owner=params.spark_user,
group=params.spark_group,
content=params.spark_log4j_properties,
mode=0644,
)
#create metrics.properties in etc/conf dir
File(os.path.join(params.spark_conf, 'metrics.properties'),
owner=params.spark_user,
group=params.spark_group,
content=InlineTemplate(params.spark_metrics_properties),
mode=0644
)
if params.is_hive_installed:
XmlConfig("hive-site.xml",
conf_dir=params.spark_conf,
configurations=params.spark_hive_properties,
owner=params.spark_user,
group=params.spark_group,
mode=0644)
if params.has_spark_thriftserver:
spark2_thrift_sparkconf = dict(params.config['configurations']['spark2-thrift-sparkconf'])
if params.security_enabled and 'spark.yarn.principal' in spark2_thrift_sparkconf:
spark2_thrift_sparkconf['spark.yarn.principal'] = spark2_thrift_sparkconf['spark.yarn.principal'].replace('_HOST', socket.getfqdn().lower())
PropertiesFile(params.spark_thrift_server_conf_file,
properties = spark2_thrift_sparkconf,
owner = params.hive_user,
group = params.user_group,
key_value_delimiter = " ",
mode=0644
)
effective_version = params.version if upgrade_type is not None else params.stack_version_formatted
if effective_version:
effective_version = format_stack_version(effective_version)
if params.spark_thrift_fairscheduler_content and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version):
# create spark-thrift-fairscheduler.xml
File(os.path.join(params.spark_conf,"spark-thrift-fairscheduler.xml"),
owner=params.spark_user,
group=params.spark_group,
mode=0755,
content=InlineTemplate(params.spark_thrift_fairscheduler_content)
)
def ams(name=None, action=None):
import params
if name == 'collector':
Directory(
params.ams_collector_conf_dir,
owner=params.ams_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True,
)
Directory(params.ams_checkpoint_dir,
owner=params.ams_user,
group=params.user_group,
cd_access="a",
create_parents=True,
recursive_ownership=True)
new_ams_site = {}
new_ams_site.update(params.config['configurations']['ams-site'])
if params.clusterHostInfoDict:
master_components = []
slave_components = []
components = dict(params.clusterHostInfoDict).keys()
known_slave_components = [
"nodemanager", "metrics_monitor", "datanode",
"hbase_regionserver"
]
for component in components:
if component and component.endswith("_hosts"):
component_name = component[:-6]
elif component and component.endswith("_host"):
component_name = component[:-5]
else:
continue
if component_name in known_slave_components:
slave_components.append(component_name)
else:
master_components.append(component_name)
if slave_components:
new_ams_site[
'timeline.metrics.initial.configured.slave.components'] = ",".join(
slave_components)
if master_components:
if 'ambari_server' not in master_components:
master_components.append('ambari_server')
new_ams_site[
'timeline.metrics.initial.configured.master.components'] = ",".join(
master_components)
hbase_total_heapsize_with_trailing_m = params.hbase_heapsize
hbase_total_heapsize = int(
hbase_total_heapsize_with_trailing_m[:-1]) * 1024 * 1024
new_ams_site['hbase_total_heapsize'] = hbase_total_heapsize
XmlConfig(
"ams-site.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=new_ams_site,
configuration_attributes=params.config['configurationAttributes']
['ams-site'],
owner=params.ams_user,
group=params.user_group)
XmlConfig(
"ssl-server.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=params.config['configurations']['ams-ssl-server'],
configuration_attributes=params.config['configurationAttributes']
['ams-ssl-server'],
owner=params.ams_user,
group=params.user_group)
merged_ams_hbase_site = {}
merged_ams_hbase_site.update(
params.config['configurations']['ams-hbase-site'])
if params.security_enabled:
merged_ams_hbase_site.update(
params.config['configurations']['ams-hbase-security-site'])
# Add phoenix client side overrides
merged_ams_hbase_site['phoenix.query.maxGlobalMemoryPercentage'] = str(
params.phoenix_max_global_mem_percent)
merged_ams_hbase_site[
'phoenix.spool.directory'] = params.phoenix_client_spool_dir
XmlConfig(
"hbase-site.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=merged_ams_hbase_site,
configuration_attributes=params.config['configurationAttributes']
['ams-hbase-site'],
owner=params.ams_user,
group=params.user_group)
if params.security_enabled:
TemplateConfig(os.path.join(params.hbase_conf_dir,
"ams_collector_jaas.conf"),
owner=params.ams_user,
template_tag=None)
if (params.log4j_props != None):
File(format("{params.ams_collector_conf_dir}/log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.ams_user,
content=InlineTemplate(params.log4j_props))
File(format("{ams_collector_conf_dir}/ams-env.sh"),
owner=params.ams_user,
content=InlineTemplate(params.ams_env_sh_template))
Directory(
params.ams_collector_log_dir,
owner=params.ams_user,
group=params.user_group,
cd_access="a",
create_parents=True,
mode=0755,
)
Directory(
params.ams_collector_pid_dir,
owner=params.ams_user,
group=params.user_group,
cd_access="a",
create_parents=True,
mode=0755,
)
# Hack to allow native HBase libs to be included for embedded hbase
File(os.path.join(params.ams_hbase_home_dir, "bin", "hadoop"),
owner=params.ams_user,
mode=0755)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
# Setting up security limits
File(os.path.join(params.limits_conf_dir, 'ams.conf'),
owner='root',
group='root',
mode=0644,
content=Template("ams.conf.j2"))
# Phoenix spool file dir if not /tmp
if not os.path.exists(params.phoenix_client_spool_dir):
Directory(params.phoenix_client_spool_dir,
owner=params.ams_user,
mode=0755,
group=params.user_group,
cd_access="a",
create_parents=True)
pass
if not params.is_local_fs_rootdir and params.is_ams_distributed:
# Configuration needed to support NN HA
XmlConfig(
"hdfs-site.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.
config['configurationAttributes']['hdfs-site'],
owner=params.ams_user,
group=params.user_group,
mode=0644)
XmlConfig(
"hdfs-site.xml",
conf_dir=params.hbase_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.
config['configurationAttributes']['hdfs-site'],
owner=params.ams_user,
group=params.user_group,
mode=0644)
# Remove spnego configs from core-site if platform does not have python-kerberos library
truncated_core_site = {}
truncated_core_site.update(
params.config['configurations']['core-site'])
if is_spnego_enabled(params):
truncated_core_site.pop('hadoop.http.authentication.type')
truncated_core_site.pop('hadoop.http.filter.initializers')
# if there is the viewFS mount table content, create separate xml config and include in in the core-sites
# else just create core-sites
if params.mount_table_content:
XmlConfig("core-site.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=truncated_core_site,
configuration_attributes=params.
config['configurationAttributes']['core-site'],
owner=params.ams_user,
group=params.user_group,
mode=0644,
xml_include_file=os.path.join(
params.ams_collector_conf_dir,
params.xml_inclusion_file_name))
File(os.path.join(params.ams_collector_conf_dir,
params.xml_inclusion_file_name),
owner=params.ams_user,
group=params.user_group,
content=params.mount_table_content,
mode=0644)
XmlConfig("core-site.xml",
conf_dir=params.hbase_conf_dir,
configurations=truncated_core_site,
configuration_attributes=params.
config['configurationAttributes']['core-site'],
owner=params.ams_user,
group=params.user_group,
mode=0644,
xml_include_file=os.path.join(
params.hbase_conf_dir,
params.xml_inclusion_file_name))
File(os.path.join(params.hbase_conf_dir,
params.xml_inclusion_file_name),
owner=params.ams_user,
group=params.user_group,
content=params.mount_table_content,
mode=0644)
else:
XmlConfig("core-site.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=truncated_core_site,
configuration_attributes=params.
config['configurationAttributes']['core-site'],
owner=params.ams_user,
group=params.user_group,
mode=0644)
XmlConfig("core-site.xml",
conf_dir=params.hbase_conf_dir,
configurations=truncated_core_site,
configuration_attributes=params.
config['configurationAttributes']['core-site'],
owner=params.ams_user,
group=params.user_group,
mode=0644)
if params.metric_collector_https_enabled:
export_ca_certs(params.ams_collector_conf_dir)
pass
elif name == 'monitor':
# TODO Uncomment when SPNEGO support has been added to AMS service check and Grafana.
if is_spnego_enabled(params) and is_redhat_centos_6_plus():
try:
import kerberos
except ImportError:
raise ImportError(
"python-kerberos package need to be installed to run AMS in SPNEGO mode"
)
Directory(params.ams_monitor_conf_dir,
owner=params.ams_user,
group=params.user_group,
create_parents=True)
Directory(params.ams_monitor_log_dir,
owner=params.ams_user,
group=params.user_group,
mode=0755,
create_parents=True)
if params.host_in_memory_aggregation and params.log4j_props is not None:
File(format("{params.ams_monitor_conf_dir}/log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.ams_user,
content=InlineTemplate(params.log4j_props))
XmlConfig(
"ams-site.xml",
conf_dir=params.ams_monitor_conf_dir,
configurations=params.config['configurations']['ams-site'],
configuration_attributes=params.
config['configurationAttributes']['ams-site'],
owner=params.ams_user,
group=params.user_group)
XmlConfig("ssl-server.xml",
conf_dir=params.ams_monitor_conf_dir,
configurations=params.config['configurations']
['ams-ssl-server'],
configuration_attributes=params.
config['configurationAttributes']['ams-ssl-server'],
owner=params.ams_user,
group=params.user_group)
pass
Execute(
format(
"{sudo} chown -R {ams_user}:{user_group} {ams_monitor_log_dir}"
))
Directory(params.ams_monitor_pid_dir,
owner=params.ams_user,
group=params.user_group,
cd_access="a",
mode=0755,
create_parents=True)
Directory(format("{ams_monitor_dir}/psutil/build"),
owner=params.ams_user,
group=params.user_group,
cd_access="a",
create_parents=True)
Execute(
format(
"{sudo} chown -R {ams_user}:{user_group} {ams_monitor_dir}"))
TemplateConfig(format("{ams_monitor_conf_dir}/metric_monitor.ini"),
owner=params.ams_user,
group=params.user_group,
template_tag=None)
TemplateConfig(format("{ams_monitor_conf_dir}/metric_groups.conf"),
owner=params.ams_user,
group=params.user_group,
template_tag=None)
File(format("{ams_monitor_conf_dir}/ams-env.sh"),
owner=params.ams_user,
content=InlineTemplate(params.ams_env_sh_template))
if params.metric_collector_https_enabled or params.is_aggregation_https_enabled:
export_ca_certs(params.ams_monitor_conf_dir)
pass
elif name == 'grafana':
ams_grafana_directories = [
params.ams_grafana_conf_dir, params.ams_grafana_log_dir,
params.ams_grafana_data_dir, params.ams_grafana_pid_dir
]
for ams_grafana_directory in ams_grafana_directories:
Directory(ams_grafana_directory,
owner=params.ams_user,
group=params.user_group,
mode=0755,
create_parents=True,
recursive_ownership=True)
File(format("{ams_grafana_conf_dir}/ams-grafana-env.sh"),
owner=params.ams_user,
group=params.user_group,
content=InlineTemplate(params.ams_grafana_env_sh_template))
File(format("{ams_grafana_conf_dir}/ams-grafana.ini"),
owner=params.ams_user,
group=params.user_group,
content=InlineTemplate(params.ams_grafana_ini_template),
mode=0600)
if action != 'stop':
for dir in ams_grafana_directories:
Execute(('chown', '-R', params.ams_user, dir), sudo=True)
if params.metric_collector_https_enabled:
export_ca_certs(params.ams_grafana_conf_dir)
pass
generate_logfeeder_input_config(
'ambari-metrics',
Template("input.config-ambari-metrics.json.j2",
extra_imports=[default]))
def kafka(upgrade_type=None):
import params
ensure_base_directories()
kafka_server_config = mutable_config_dict(
params.config['configurations']['kafka-broker'])
# This still has an issue of hostnames being alphabetically out-of-order for broker.id in HDP-2.2.
# Starting in HDP 2.3, Kafka handles the generation of broker.id so Ambari doesn't have to.
effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(
params.version)
Logger.info(format("Effective stack version: {effective_version}"))
# listeners and advertised.listeners are only added in 2.3.0.0 onwards.
if effective_version is not None and effective_version != "" and \
check_stack_feature(StackFeature.KAFKA_LISTENERS, effective_version):
listeners = kafka_server_config['listeners'].replace(
"localhost", params.hostname)
Logger.info(format("Kafka listeners: {listeners}"))
kafka_server_config['listeners'] = listeners
if params.kerberos_security_enabled and params.kafka_kerberos_enabled:
Logger.info("Kafka kerberos security is enabled.")
if "SASL" not in listeners:
listeners = kafka_server_config['listeners']
listeners = re.sub(r"(^|\b)PLAINTEXT://", "SASL_PLAINTEXT://",
listeners)
listeners = re.sub(r"(^|\b)PLAINTEXTSASL://",
"SASL_PLAINTEXT://", listeners)
listeners = re.sub(r"(^|\b)SSL://", "SASL_SSL://", listeners)
kafka_server_config['listeners'] = listeners
kafka_server_config['advertised.listeners'] = listeners
Logger.info(format("Kafka advertised listeners: {listeners}"))
elif 'advertised.listeners' in kafka_server_config:
advertised_listeners = kafka_server_config[
'advertised.listeners'].replace("localhost", params.hostname)
kafka_server_config['advertised.listeners'] = advertised_listeners
Logger.info(
format("Kafka advertised listeners: {advertised_listeners}"))
else:
kafka_server_config['host.name'] = params.hostname
if params.has_metric_collector:
kafka_server_config[
'kafka.timeline.metrics.hosts'] = params.ams_collector_hosts
kafka_server_config[
'kafka.timeline.metrics.port'] = params.metric_collector_port
kafka_server_config[
'kafka.timeline.metrics.protocol'] = params.metric_collector_protocol
kafka_server_config[
'kafka.timeline.metrics.truststore.path'] = params.metric_truststore_path
kafka_server_config[
'kafka.timeline.metrics.truststore.type'] = params.metric_truststore_type
kafka_server_config[
'kafka.timeline.metrics.truststore.password'] = params.metric_truststore_password
kafka_data_dir = kafka_server_config['log.dirs']
kafka_data_dirs = filter(None, kafka_data_dir.split(","))
rack = "/default-rack"
i = 0
if len(params.all_racks) > 0:
for host in params.all_hosts:
if host == params.hostname:
rack = params.all_racks[i]
break
i = i + 1
Directory(
kafka_data_dirs,
mode=0755,
cd_access='a',
owner=params.kafka_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True,
)
PropertiesFile(
"server.properties",
mode=0640,
dir=params.conf_dir,
properties=kafka_server_config,
owner=params.kafka_user,
group=params.user_group,
)
File(format("{conf_dir}/kafka-env.sh"),
owner=params.kafka_user,
content=InlineTemplate(params.kafka_env_sh_template))
if (params.log4j_props != None):
File(format("{conf_dir}/log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.kafka_user,
content=InlineTemplate(params.log4j_props))
if (params.kerberos_security_enabled and
params.kafka_kerberos_enabled) or params.kafka_other_sasl_enabled:
if params.kafka_jaas_conf_template:
File(format("{conf_dir}/kafka_jaas.conf"),
owner=params.kafka_user,
content=InlineTemplate(params.kafka_jaas_conf_template))
else:
TemplateConfig(format("{conf_dir}/kafka_jaas.conf"),
owner=params.kafka_user)
if params.kafka_client_jaas_conf_template:
File(format("{conf_dir}/kafka_client_jaas.conf"),
owner=params.kafka_user,
content=InlineTemplate(
params.kafka_client_jaas_conf_template))
else:
TemplateConfig(format("{conf_dir}/kafka_client_jaas.conf"),
owner=params.kafka_user)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'kafka.conf'),
owner='root',
group='root',
mode=0644,
content=Template("kafka.conf.j2"))
File(os.path.join(params.conf_dir, 'tools-log4j.properties'),
owner='root',
group='root',
mode=0644,
content=Template("tools-log4j.properties.j2"))
generate_logfeeder_input_config(
'kafka', Template("input.config-kafka.json.j2",
extra_imports=[default]))
setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir)
setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)
def setup_ranger_admin(upgrade_type=None):
import params
if upgrade_type is None:
upgrade_type = Script.get_upgrade_type(
default("/commandParams/upgrade_type", ""))
ranger_home = params.ranger_home
ranger_conf = params.ranger_conf
Directory(ranger_conf,
owner=params.unix_user,
group=params.unix_group,
create_parents=True)
copy_jdbc_connector(ranger_home)
File(
format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}/{check_db_connection_jar_name}")),
mode=0644,
)
generate_logfeeder_input_config(
'ranger',
Template("input.config-ranger.json.j2", extra_imports=[default]))
cp = format("{check_db_connection_jar}")
if params.db_flavor.lower() == 'sqla':
cp = cp + os.pathsep + format("{ranger_home}/ews/lib/sajdbc4.jar")
else:
cp = cp + os.pathsep + format("{driver_curl_target}")
cp = cp + os.pathsep + format("{ranger_home}/ews/lib/*")
db_connection_check_command = format(
"{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_jdbc_connection_url}' {ranger_db_user} {ranger_db_password!p} {ranger_jdbc_driver}"
)
env_dict = {}
if params.db_flavor.lower() == 'sqla':
env_dict = {'LD_LIBRARY_PATH': params.ld_lib_path}
Execute(db_connection_check_command,
path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
tries=5,
try_sleep=10,
environment=env_dict)
Execute(
('ln', '-sf', format('{ranger_home}/ews/webapp/WEB-INF/classes/conf'),
format('{ranger_home}/conf')),
not_if=format("ls {ranger_home}/conf"),
only_if=format("ls {ranger_home}/ews/webapp/WEB-INF/classes/conf"),
sudo=True)
if upgrade_type is not None:
src_file = format(
'{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/ranger-admin-default-site.xml'
)
dst_file = format('{ranger_home}/conf/ranger-admin-default-site.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
src_file = format(
'{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/security-applicationContext.xml'
)
dst_file = format('{ranger_home}/conf/security-applicationContext.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
Directory(
format('{ranger_home}/'),
owner=params.unix_user,
group=params.unix_group,
recursive_ownership=True,
)
Directory(params.ranger_pid_dir,
mode=0755,
owner=params.unix_user,
group=params.user_group,
cd_access="a",
create_parents=True)
Directory(params.admin_log_dir,
owner=params.unix_user,
group=params.unix_group,
create_parents=True,
cd_access='a',
mode=0755)
if os.path.isfile(params.ranger_admin_default_file):
File(params.ranger_admin_default_file,
owner=params.unix_user,
group=params.unix_group)
else:
Logger.warning(
'Required file {0} does not exist, copying the file to {1} path'.
format(params.ranger_admin_default_file, ranger_conf))
src_file = format(
'{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/ranger-admin-default-site.xml'
)
dst_file = format('{ranger_home}/conf/ranger-admin-default-site.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
File(params.ranger_admin_default_file,
owner=params.unix_user,
group=params.unix_group)
if os.path.isfile(params.security_app_context_file):
File(params.security_app_context_file,
owner=params.unix_user,
group=params.unix_group)
else:
Logger.warning(
'Required file {0} does not exist, copying the file to {1} path'.
format(params.security_app_context_file, ranger_conf))
src_file = format(
'{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/security-applicationContext.xml'
)
dst_file = format('{ranger_home}/conf/security-applicationContext.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
File(params.security_app_context_file,
owner=params.unix_user,
group=params.unix_group)
if default(
"/configurations/ranger-admin-site/ranger.authentication.method",
"") == 'PAM':
d = '/etc/pam.d'
if os.path.isdir(d):
if os.path.isfile(os.path.join(d, 'ranger-admin')):
Logger.info('ranger-admin PAM file already exists.')
else:
File(format('{d}/ranger-admin'),
content=Template('ranger_admin_pam.j2'),
owner=params.unix_user,
group=params.unix_group,
mode=0644)
if os.path.isfile(os.path.join(d, 'ranger-remote')):
Logger.info('ranger-remote PAM file already exists.')
else:
File(format('{d}/ranger-remote'),
content=Template('ranger_remote_pam.j2'),
owner=params.unix_user,
group=params.unix_group,
mode=0644)
else:
Logger.error(
"Unable to use PAM authentication, /etc/pam.d/ directory does not exist."
)
# remove plain-text password from xml configs
ranger_admin_site_copy = {}
ranger_admin_site_copy.update(
params.config['configurations']['ranger-admin-site'])
for prop in params.ranger_admin_password_properties:
if prop in ranger_admin_site_copy:
ranger_admin_site_copy[prop] = "_"
if 'ranger.ha.spnego.kerberos.keytab' in ranger_admin_site_copy:
ranger_admin_site_copy[
'ranger.spnego.kerberos.keytab'] = ranger_admin_site_copy[
'ranger.ha.spnego.kerberos.keytab']
XmlConfig("ranger-admin-site.xml",
conf_dir=ranger_conf,
configurations=ranger_admin_site_copy,
configuration_attributes=params.config['configurationAttributes']
['ranger-admin-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
Directory(
os.path.join(ranger_conf, 'ranger_jaas'),
mode=0700,
owner=params.unix_user,
group=params.unix_group,
)
if params.stack_supports_ranger_log4j:
File(format('{ranger_home}/ews/webapp/WEB-INF/log4j.properties'),
owner=params.unix_user,
group=params.unix_group,
content=InlineTemplate(params.admin_log4j),
mode=0644)
do_keystore_setup(upgrade_type=upgrade_type)
create_core_site_xml(ranger_conf)
if params.stack_supports_ranger_kerberos:
if params.is_hbase_ha_enabled and params.ranger_hbase_plugin_enabled:
XmlConfig(
"hbase-site.xml",
conf_dir=ranger_conf,
configurations=params.config['configurations']['hbase-site'],
configuration_attributes=params.
config['configurationAttributes']['hbase-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
if params.is_namenode_ha_enabled and params.ranger_hdfs_plugin_enabled:
XmlConfig(
"hdfs-site.xml",
conf_dir=ranger_conf,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.
config['configurationAttributes']['hdfs-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
File(format("{ranger_conf}/ranger-admin-env.sh"),
content=InlineTemplate(params.ranger_env_content),
owner=params.unix_user,
group=params.unix_group,
mode=0755)
def setup_hiveserver2():
import params
File(params.start_hiveserver2_path,
mode=0755,
content=Template(format('{start_hiveserver2_script}')))
File(os.path.join(params.hive_server_conf_dir,
"hadoop-metrics2-hiveserver2.properties"),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-hiveserver2.properties.j2"),
mode=0600)
XmlConfig(
"hiveserver2-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']['hiveserver2-site'],
configuration_attributes=params.config['configurationAttributes']
['hiveserver2-site'],
owner=params.hive_user,
group=params.user_group,
mode=0600)
# ****** Begin Copy Tarballs ******
# *********************************
# if copy tarball to HDFS feature supported copy mapreduce.tar.gz and tez.tar.gz to HDFS
if params.stack_version_formatted_major and check_stack_feature(
StackFeature.COPY_TARBALL_TO_HDFS,
params.stack_version_formatted_major):
copy_to_hdfs("mapreduce",
params.user_group,
params.hdfs_user,
skip=params.sysprep_skip_copy_tarballs_hdfs)
copy_to_hdfs("tez",
params.user_group,
params.hdfs_user,
skip=params.sysprep_skip_copy_tarballs_hdfs)
# Always copy pig.tar.gz and hive.tar.gz using the appropriate mode.
# This can use a different source and dest location to account
copy_to_hdfs("pig",
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=params.pig_tar_source,
custom_dest_file=params.pig_tar_dest_file,
skip=params.sysprep_skip_copy_tarballs_hdfs)
copy_to_hdfs("hive",
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=params.hive_tar_source,
custom_dest_file=params.hive_tar_dest_file,
skip=params.sysprep_skip_copy_tarballs_hdfs)
wildcard_tarballs = ["sqoop", "hadoop_streaming"]
for tarball_name in wildcard_tarballs:
source_file_pattern = eval("params." + tarball_name + "_tar_source")
dest_dir = eval("params." + tarball_name + "_tar_dest_dir")
if source_file_pattern is None or dest_dir is None:
continue
source_files = glob.glob(
source_file_pattern) if "*" in source_file_pattern else [
source_file_pattern
]
for source_file in source_files:
src_filename = os.path.basename(source_file)
dest_file = os.path.join(dest_dir, src_filename)
copy_to_hdfs(tarball_name,
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=source_file,
custom_dest_file=dest_file,
skip=params.sysprep_skip_copy_tarballs_hdfs)
# ******* End Copy Tarballs *******
# *********************************
# if warehouse directory is in DFS
if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(
params.default_fs).scheme:
if not is_empty(params.tez_hook_proto_base_directory):
params.HdfsResource(params.tez_hook_proto_base_directory,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01755)
if not is_empty(params.hive_hook_proto_base_directory):
params.HdfsResource(params.hive_hook_proto_base_directory,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
dag_meta = params.tez_hook_proto_base_directory + "dag_meta"
params.HdfsResource(dag_meta,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
dag_data = params.tez_hook_proto_base_directory + "dag_data"
params.HdfsResource(dag_data,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
app_data = params.tez_hook_proto_base_directory + "app_data"
params.HdfsResource(app_data,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
if not is_empty(params.hive_exec_scratchdir) and not urlparse(
params.hive_exec_scratchdir).path.startswith("/tmp"):
params.HdfsResource(
params.hive_exec_scratchdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.hdfs_user,
mode=0777
) # Hive expects this dir to be writeable by everyone as it is used as a temp dir
if params.hive_repl_cmrootdir is not None and params.hive_repl_cmrootdir.strip(
) != "":
params.HdfsResource(params.hive_repl_cmrootdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=01777)
if params.hive_repl_rootdir is not None and params.hive_repl_rootdir.strip(
) != "":
params.HdfsResource(params.hive_repl_rootdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=0700)
params.HdfsResource(None, action="execute")
generate_logfeeder_input_config(
'hive', Template("input.config-hive.json.j2", extra_imports=[default]))
def hbase(
name=None # 'master' or 'regionserver' or 'client'
,
action=None):
import params
Directory(
params.hbase_conf_dir,
owner=params.hbase_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True,
)
Directory(
params.hbase_tmp_dir,
owner=params.hbase_user,
cd_access="a",
create_parents=True,
recursive_ownership=True,
)
Directory(
os.path.join(params.local_dir, "jars"),
owner=params.hbase_user,
group=params.user_group,
cd_access="a",
mode=0775,
create_parents=True)
if params.hbase_wal_dir:
Directory(
params.hbase_wal_dir,
owner=params.hbase_user,
group=params.user_group,
cd_access="a",
create_parents=True,
recursive_ownership=True,
)
merged_ams_hbase_site = {}
merged_ams_hbase_site.update(
params.config['configurations']['ams-hbase-site'])
if params.security_enabled:
merged_ams_hbase_site.update(
params.config['configurations']['ams-hbase-security-site'])
if not params.is_hbase_distributed:
File(
format("{hbase_conf_dir}/core-site.xml"),
action='delete',
owner=params.hbase_user)
File(
format("{hbase_conf_dir}/hdfs-site.xml"),
action='delete',
owner=params.hbase_user)
XmlConfig(
"hbase-site.xml",
conf_dir=params.hbase_conf_dir,
configurations=merged_ams_hbase_site,
configuration_attributes=params.config['configurationAttributes']
['ams-hbase-site'],
owner=params.hbase_user,
group=params.user_group)
# Phoenix spool file dir if not /tmp
if not os.path.exists(params.phoenix_server_spool_dir):
Directory(
params.phoenix_server_spool_dir,
owner=params.ams_user,
mode=0755,
group=params.user_group,
cd_access="a",
create_parents=True)
pass
if 'ams-hbase-policy' in params.config['configurations']:
XmlConfig(
"hbase-policy.xml",
conf_dir=params.hbase_conf_dir,
configurations=params.config['configurations']['ams-hbase-policy'],
configuration_attributes=params.config['configurationAttributes']
['ams-hbase-policy'],
owner=params.hbase_user,
group=params.user_group)
# Manually overriding ownership of file installed by hadoop package
else:
File(
format("{params.hbase_conf_dir}/hbase-policy.xml"),
owner=params.hbase_user,
group=params.user_group)
File(
format("{hbase_conf_dir}/hbase-env.sh"),
owner=params.hbase_user,
content=InlineTemplate(params.hbase_env_sh_template))
# Metrics properties
File(
os.path.join(params.hbase_conf_dir,
"hadoop-metrics2-hbase.properties"),
owner=params.hbase_user,
group=params.user_group,
content=Template("hadoop-metrics2-hbase.properties.j2"))
# hbase_TemplateConfig( params.metric_prop_file_name,
# tag = 'GANGLIA-MASTER' if name == 'master' else 'GANGLIA-RS'
# )
hbase_TemplateConfig('regionservers', user=params.hbase_user)
if params.security_enabled:
hbase_TemplateConfig(
format("hbase_{name}_jaas.conf"), user=params.hbase_user)
hbase_TemplateConfig(
format("hbase_client_jaas.conf"), user=params.hbase_user)
hbase_TemplateConfig(
format("ams_zookeeper_jaas.conf"), user=params.hbase_user)
if name != "client":
Directory(
params.hbase_pid_dir,
owner=params.hbase_user,
create_parents=True,
cd_access="a",
mode=0755,
)
Directory(
params.hbase_log_dir,
owner=params.hbase_user,
create_parents=True,
cd_access="a",
mode=0755,
)
if name == "master":
if not params.is_local_fs_rootdir:
# If executing Stop All, HDFS is probably down
if action != 'stop' and not params.skip_create_hbase_root_dir:
params.HdfsResource(
params.hbase_root_dir,
type="directory",
action="create_on_execute",
owner=params.hbase_user,
mode=0775,
dfs_type=params.dfs_type)
params.HdfsResource(
params.hbase_staging_dir,
type="directory",
action="create_on_execute",
owner=params.hbase_user,
mode=0711,
dfs_type=params.dfs_type)
params.HdfsResource(None, action="execute")
if params.is_hbase_distributed:
# Workaround for status commands not aware of operating mode
File(
format("{params.hbase_pid_dir}/distributed_mode"),
action="create",
mode=0644,
owner=params.hbase_user)
pass
else:
local_root_dir = params.hbase_root_dir
# cut protocol name
if local_root_dir.startswith("file://"):
local_root_dir = local_root_dir[7:]
# otherwise assume dir name is provided as is
Directory(
local_root_dir,
owner=params.hbase_user,
cd_access="a",
create_parents=True,
recursive_ownership=True)
File(
format("{params.hbase_pid_dir}/distributed_mode"),
action="delete",
owner=params.hbase_user)
if params.hbase_log4j_props is not None:
File(
format("{params.hbase_conf_dir}/log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.hbase_user,
content=InlineTemplate(params.hbase_log4j_props))
elif os.path.exists(format("{params.hbase_conf_dir}/log4j.properties")):
File(
format("{params.hbase_conf_dir}/log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.hbase_user)
generate_logfeeder_input_config(
'ambari-metrics',
Template(
"input.config-ambari-metrics.json.j2", extra_imports=[default]))
