def create_credential_providers():
import params
if not os.path.exists(params.conf_dir):
os.makedirs(params.conf_dir)
for c in [
'data_analytics_studio-database',
'data_analytics_studio-properties',
'data_analytics_studio-security-site'
]:
update_credential_provider_path(
params.config['configurations'][c], c,
os.path.join(params.conf_dir,
c + '.jceks'), params.data_analytics_studio_user,
params.data_analytics_studio_group)
def oozie_server_specific(upgrade_type):
import params
no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user)
File(params.pid_file,
action="delete",
not_if=no_op_test
)
oozie_server_directories = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir]
Directory( oozie_server_directories,
owner = params.oozie_user,
group = params.user_group,
mode = 0755,
create_parents = True,
cd_access="a",
)
Directory(params.oozie_libext_dir,
create_parents = True,
)
hashcode_file = format("{oozie_home}/.hashcode")
skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share")
untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home)
Execute( untar_sharelib, # time-expensive
not_if = format("{no_op_test} || {skip_recreate_sharelib}"),
sudo = True,
)
configure_cmds = []
# Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params)
# Copy the first oozie ext-2.2.zip file that is found.
# This uses a list to handle the cases when migrating from some versions of BigInsights to HDP.
if source_ext_zip_paths is not None:
for source_ext_zip_path in source_ext_zip_paths:
if os.path.isfile(source_ext_zip_path):
configure_cmds.append(('cp', source_ext_zip_path, params.oozie_libext_dir))
configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}')))
Execute(configure_cmds,
not_if=no_op_test,
sudo=True,
)
break
Directory(params.oozie_webapps_conf_dir,
owner = params.oozie_user,
group = params.user_group,
recursive_ownership = True,
recursion_follow_links = True,
)
# download the database JAR
download_database_library_if_needed()
#falcon el extension
if params.has_falcon_host:
Execute(format('{sudo} cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}'),
not_if = no_op_test)
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'),
not_if = no_op_test)
if params.lzo_enabled and len(params.all_lzo_packages) > 0:
Package(params.all_lzo_packages,
retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
not_if = no_op_test,
)
prepare_war(params)
File(hashcode_file,
mode = 0644,
)
if params.stack_version_formatted and check_stack_feature(StackFeature.OOZIE_CREATE_HIVE_TEZ_CONFIGS, params.stack_version_formatted):
# Create hive-site and tez-site configs for oozie
Directory(params.hive_conf_dir,
create_parents = True,
owner = params.oozie_user,
group = params.user_group
)
if 'hive-site' in params.config['configurations']:
hive_site_config = update_credential_provider_path(params.config['configurations']['hive-site'],
'hive-site',
os.path.join(params.hive_conf_dir, 'hive-site.jceks'),
params.oozie_user,
params.user_group
)
XmlConfig("hive-site.xml",
conf_dir=params.hive_conf_dir,
configurations=hive_site_config,
configuration_attributes=params.config['configuration_attributes']['hive-site'],
owner=params.oozie_user,
group=params.user_group,
mode=0644
)
if 'tez-site' in params.config['configurations']:
XmlConfig( "tez-site.xml",
conf_dir = params.hive_conf_dir,
configurations = params.config['configurations']['tez-site'],
configuration_attributes=params.config['configuration_attributes']['tez-site'],
owner = params.oozie_user,
group = params.user_group,
mode = 0664
)
# If Atlas is also installed, need to generate Atlas Hive hook (hive-atlas-application.properties file) in directory
# {stack_root}/{current_version}/atlas/hook/hive/
# Because this is a .properties file instead of an xml file, it will not be read automatically by Oozie.
# However, should still save the file on this host so that can upload it to the Oozie Sharelib in DFS.
if has_atlas_in_cluster():
atlas_hook_filepath = os.path.join(params.hive_conf_dir, params.atlas_hook_filename)
Logger.info("Has atlas in cluster, will save Atlas Hive hook into location %s" % str(atlas_hook_filepath))
setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.oozie_user, params.user_group)
Directory(params.oozie_server_dir,
owner = params.oozie_user,
group = params.user_group,
recursive_ownership = True,
)
if params.security_enabled:
File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'),
owner=params.oozie_user,
group=params.user_group,
content=Template("zkmigrator_jaas.conf.j2")
)
def oozie(is_server=False, upgrade_type=None):
import params
if is_server:
params.HdfsResource(params.oozie_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.oozie_user,
mode=params.oozie_hdfs_user_mode
)
params.HdfsResource(None, action="execute")
Directory(params.conf_dir,
create_parents = True,
owner = params.oozie_user,
group = params.user_group
)
params.oozie_site = update_credential_provider_path(params.oozie_site,
'oozie-site',
os.path.join(params.conf_dir, 'oozie-site.jceks'),
params.oozie_user,
params.user_group
)
XmlConfig("oozie-site.xml",
conf_dir = params.conf_dir,
configurations = params.oozie_site,
configuration_attributes=params.config['configuration_attributes']['oozie-site'],
owner = params.oozie_user,
group = params.user_group,
mode = 0664
)
File(format("{conf_dir}/oozie-env.sh"),
owner=params.oozie_user,
content=InlineTemplate(params.oozie_env_sh_template),
group=params.user_group,
)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root'
)
File(os.path.join(params.limits_conf_dir, 'oozie.conf'),
owner='root',
group='root',
mode=0644,
content=Template("oozie.conf.j2")
)
if (params.log4j_props != None):
File(format("{params.conf_dir}/oozie-log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.oozie_user,
content=InlineTemplate(params.log4j_props)
)
elif (os.path.exists(format("{params.conf_dir}/oozie-log4j.properties"))):
File(format("{params.conf_dir}/oozie-log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.oozie_user
)
if params.stack_version_formatted and check_stack_feature(StackFeature.OOZIE_ADMIN_USER, params.stack_version_formatted):
File(format("{params.conf_dir}/adminusers.txt"),
mode=0644,
group=params.user_group,
owner=params.oozie_user,
content=Template('adminusers.txt.j2', oozie_admin_users=params.oozie_admin_users)
)
else:
File ( format("{params.conf_dir}/adminusers.txt"),
owner = params.oozie_user,
group = params.user_group
)
if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \
params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
params.jdbc_driver_name == "org.postgresql.Driver" or \
params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver":
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
)
pass
oozie_ownership()
if is_server:
oozie_server_specific(upgrade_type)
def setup_logsearch():
import params
Directory([params.logsearch_log_dir, params.logsearch_pid_dir],
mode=0755,
cd_access='a',
owner=params.logsearch_user,
group=params.user_group,
create_parents=True
)
Directory([params.logsearch_dir, params.logsearch_server_conf, params.logsearch_config_set_dir],
mode=0755,
cd_access='a',
owner=params.logsearch_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True
)
Directory(params.logsearch_server_keys_folder,
cd_access='a',
mode=0755,
owner=params.logsearch_user,
group=params.user_group)
File(params.logsearch_log,
mode=0644,
owner=params.logsearch_user,
group=params.user_group,
content=''
)
params.logsearch_env_config = update_credential_provider_path(params.logsearch_env_config,
'logsearch-env',
params.logsearch_env_jceks_file,
params.logsearch_user,
params.user_group
)
params.logsearch_properties[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'jceks://file' + params.logsearch_env_jceks_file
PropertiesFile(format("{logsearch_server_conf}/logsearch.properties"),
properties=params.logsearch_properties
)
File(format("{logsearch_server_conf}/HadoopServiceConfig.json"),
content=Template("HadoopServiceConfig.json.j2"),
owner=params.logsearch_user,
group=params.user_group
)
File(format("{logsearch_server_conf}/log4j.xml"),
content=InlineTemplate(params.logsearch_app_log4j_content),
owner=params.logsearch_user,
group=params.user_group
)
File(format("{logsearch_server_conf}/logsearch-env.sh"),
content=InlineTemplate(params.logsearch_env_content),
mode=0755,
owner=params.logsearch_user,
group=params.user_group
)
File(format("{logsearch_server_conf}/logsearch-admin.json"),
content=InlineTemplate(params.logsearch_admin_content),
owner=params.logsearch_user,
group=params.user_group
)
File(format("{logsearch_config_set_dir}/hadoop_logs/conf/solrconfig.xml"),
content=InlineTemplate(params.logsearch_service_logs_solrconfig_content),
owner=params.logsearch_user,
group=params.user_group
)
File(format("{logsearch_config_set_dir}/audit_logs/conf/solrconfig.xml"),
content=InlineTemplate(params.logsearch_audit_logs_solrconfig_content),
owner=params.logsearch_user,
group=params.user_group
)
if params.security_enabled:
File(format("{logsearch_jaas_file}"),
content=Template("logsearch_jaas.conf.j2"),
owner=params.logsearch_user
)
Execute(("chmod", "-R", "ugo+r", format("{logsearch_server_conf}/solr_configsets")),
sudo=True
)
check_znode()
if params.security_enabled and not params.logsearch_use_external_solr:
solr_cloud_util.add_solr_roles(params.config,
roles = [params.infra_solr_role_logsearch, params.infra_solr_role_ranger_admin, params.infra_solr_role_dev],
new_service_principals = [params.logsearch_kerberos_principal])
solr_cloud_util.add_solr_roles(params.config,
roles = [params.infra_solr_role_logfeeder, params.infra_solr_role_dev],
new_service_principals = [params.logfeeder_kerberos_principal])
def hive(name=None):
import params
hive_client_conf_path = format("{stack_root}/current/{component_directory}/conf")
# Permissions 644 for conf dir (client) files, and 600 for conf.server
mode_identified = 0644 if params.hive_config_dir == hive_client_conf_path else 0600
Directory(params.hive_etc_dir_prefix,
mode=0755
)
# We should change configurations for client as well as for server.
# The reason is that stale-configs are service-level, not component.
Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list))
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
params.hive_site_config = update_credential_provider_path(params.hive_site_config,
'hive-site',
os.path.join(params.hive_conf_dir, 'hive-site.jceks'),
params.hive_user,
params.user_group
)
XmlConfig("hive-site.xml",
conf_dir=params.hive_config_dir,
configurations=params.hive_site_config,
configuration_attributes=params.config['configuration_attributes']['hive-site'],
owner=params.hive_user,
group=params.user_group,
mode=mode_identified)
# Generate atlas-application.properties.xml file
if params.enable_atlas_hook:
atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename)
setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group)
File(format("{hive_config_dir}/hive-env.sh"),
owner=params.hive_user,
group=params.user_group,
content=InlineTemplate(params.hive_env_sh_template),
mode=mode_identified
)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents = True,
owner='root',
group='root'
)
File(os.path.join(params.limits_conf_dir, 'hive.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hive.conf.j2")
)
if params.security_enabled:
File(os.path.join(params.hive_config_dir, 'zkmigrator_jaas.conf'),
owner=params.hive_user,
group=params.user_group,
content=Template("zkmigrator_jaas.conf.j2")
)
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
mode = 0644,
)
if name != "client":
setup_non_client()
if name == 'hiveserver2':
setup_hiveserver2()
if name == 'metastore':
setup_metastore()
def hive_interactive(name=None):
import params
MB_TO_BYTES = 1048576
# if warehouse directory is in DFS
if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(
params.default_fs).scheme:
# Create Hive Metastore Warehouse Dir
params.HdfsResource(params.hive_apps_whs_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=params.hive_apps_whs_mode)
else:
Logger.info(
format(
"Not creating warehouse directory '{hive_apps_whs_dir}', as the location is not in DFS."
))
# Create Hive User Dir
params.HdfsResource(params.hive_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=params.hive_hdfs_user_mode)
# list of properties that should be excluded from the config
# this approach is a compromise against adding a dedicated config
# type for hive_server_interactive or needed config groups on a
# per component basis
exclude_list = ['hive.enforce.bucketing', 'hive.enforce.sorting']
# List of configs to be excluded from hive2 client, but present in Hive2 server.
exclude_list_for_hive2_client = [
'javax.jdo.option.ConnectionPassword',
'hadoop.security.credential.provider.path'
]
# Copy Tarballs in HDFS.
if params.stack_version_formatted_major and check_stack_feature(
StackFeature.ROLLING_UPGRADE,
params.stack_version_formatted_major):
resource_created = copy_to_hdfs(
"tez_hive2",
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
skip=params.sysprep_skip_copy_tarballs_hdfs)
if resource_created:
params.HdfsResource(None, action="execute")
Directory(params.hive_interactive_etc_dir_prefix, mode=0755)
Logger.info("Directories to fill with configs: %s" %
str(params.hive_conf_dirs_list))
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
'''
As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml,
we need to merge hive/hive-site.xml and hive2/hive-site.xml and store it in hive2/hive-site.xml.
'''
merged_hive_interactive_site = {}
merged_hive_interactive_site.update(
params.config['configurations']['hive-site'])
merged_hive_interactive_site.update(
params.config['configurations']['hive-interactive-site'])
for item in exclude_list:
if item in merged_hive_interactive_site.keys():
del merged_hive_interactive_site[item]
'''
Config 'hive.llap.io.memory.size' calculated value in stack_advisor is in MB as of now. We need to
convert it to bytes before we write it down to config file.
'''
if 'hive.llap.io.memory.size' in merged_hive_interactive_site.keys():
hive_llap_io_mem_size_in_mb = merged_hive_interactive_site.get(
"hive.llap.io.memory.size")
hive_llap_io_mem_size_in_bytes = long(
hive_llap_io_mem_size_in_mb) * MB_TO_BYTES
merged_hive_interactive_site[
'hive.llap.io.memory.size'] = hive_llap_io_mem_size_in_bytes
Logger.info(
"Converted 'hive.llap.io.memory.size' value from '{0} MB' to '{1} Bytes' before writing "
"it to config file.".format(hive_llap_io_mem_size_in_mb,
hive_llap_io_mem_size_in_bytes))
'''
Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook',
which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed.
'''
# Generate atlas-application.properties.xml file
if params.enable_atlas_hook and params.stack_supports_atlas_hook_for_hive_interactive:
Logger.info("Setup for Atlas Hive2 Hook started.")
atlas_hook_filepath = os.path.join(
params.hive_server_interactive_conf_dir,
params.atlas_hook_filename)
setup_atlas_hook(SERVICE.HIVE,
params.hive_atlas_application_properties,
atlas_hook_filepath, params.hive_user,
params.user_group)
Logger.info("Setup for Atlas Hive2 Hook done.")
else:
# Required for HDP 2.5 stacks
Logger.info(
"Skipping setup for Atlas Hook, as it is disabled/ not supported.")
remove_atlas_hook_if_exists(merged_hive_interactive_site)
'''
As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml,
we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml.
'''
merged_tez_interactive_site = {}
if 'tez-site' in params.config['configurations']:
merged_tez_interactive_site.update(
params.config['configurations']['tez-site'])
Logger.info(
"Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'."
)
else:
Logger.error(
"Tez's 'tez-site' couldn't be retrieved from passed-in configurations."
)
merged_tez_interactive_site.update(
params.config['configurations']['tez-interactive-site'])
XmlConfig("tez-site.xml",
conf_dir=params.tez_interactive_config_dir,
configurations=merged_tez_interactive_site,
configuration_attributes=params.config['configurationAttributes']
['tez-interactive-site'],
owner=params.tez_interactive_user,
group=params.user_group,
mode=0664)
'''
Merge properties from hiveserver2-interactive-site into hiveserver2-site
'''
merged_hiveserver2_interactive_site = {}
if 'hiveserver2-site' in params.config['configurations']:
merged_hiveserver2_interactive_site.update(
params.config['configurations']['hiveserver2-site'])
Logger.info(
"Retrieved 'hiveserver2-site' for merging with 'hiveserver2-interactive-site'."
)
else:
Logger.error(
"'hiveserver2-site' couldn't be retrieved from passed-in configurations."
)
merged_hiveserver2_interactive_site.update(
params.config['configurations']['hiveserver2-interactive-site'])
# Create config files under /etc/hive2/conf and /etc/hive2/conf/conf.server:
# hive-site.xml
# hive-env.sh
# llap-daemon-log4j2.properties
# llap-cli-log4j2.properties
# hive-log4j2.properties
# hive-exec-log4j2.properties
# beeline-log4j2.properties
hive2_conf_dirs_list = params.hive_conf_dirs_list
hive2_client_conf_path = format(
"{stack_root}/current/{component_directory}/conf")
# Making copy of 'merged_hive_interactive_site' in 'merged_hive_interactive_site_copy', and deleting 'javax.jdo.option.ConnectionPassword'
# config from there, as Hive2 client shouldn't have that config.
merged_hive_interactive_site_copy = merged_hive_interactive_site.copy()
for item in exclude_list_for_hive2_client:
if item in merged_hive_interactive_site.keys():
del merged_hive_interactive_site_copy[item]
for conf_dir in hive2_conf_dirs_list:
mode_identified = 0644 if conf_dir == hive2_client_conf_path else 0600
if conf_dir == hive2_client_conf_path:
XmlConfig(
"hive-site.xml",
conf_dir=conf_dir,
configurations=merged_hive_interactive_site_copy,
configuration_attributes=params.
config['configurationAttributes']['hive-interactive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
else:
merged_hive_interactive_site = update_credential_provider_path(
merged_hive_interactive_site, 'hive-site',
os.path.join(conf_dir, 'hive-site.jceks'), params.hive_user,
params.user_group)
XmlConfig(
"hive-site.xml",
conf_dir=conf_dir,
configurations=merged_hive_interactive_site,
configuration_attributes=params.
config['configurationAttributes']['hive-interactive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0600)
XmlConfig(
"hiveserver2-site.xml",
conf_dir=conf_dir,
configurations=merged_hiveserver2_interactive_site,
configuration_attributes=params.config['configurationAttributes']
['hiveserver2-interactive-site'],
owner=params.hive_user,
group=params.user_group,
mode=mode_identified)
hive_server_interactive_conf_dir = conf_dir
File(format("{hive_server_interactive_conf_dir}/hive-env.sh"),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=InlineTemplate(params.hive_interactive_env_sh_template))
llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.llap_daemon_log4j))
llap_cli_log4j2_filename = 'llap-cli-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.llap_cli_log4j2))
hive_log4j2_filename = 'hive-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.hive_log4j2))
hive_exec_log4j2_filename = 'hive-exec-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.hive_exec_log4j2))
beeline_log4j2_filename = 'beeline-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{beeline_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.beeline_log4j2))
File(os.path.join(hive_server_interactive_conf_dir,
"hadoop-metrics2-hiveserver2.properties"),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=Template("hadoop-metrics2-hiveserver2.properties.j2"))
File(format(
"{hive_server_interactive_conf_dir}/hadoop-metrics2-llapdaemon.properties"
),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=Template("hadoop-metrics2-llapdaemon.j2"))
File(format(
"{hive_server_interactive_conf_dir}/hadoop-metrics2-llaptaskscheduler.properties"
),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=Template("hadoop-metrics2-llaptaskscheduler.j2"))
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'hive.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hive.conf.j2"))
if not os.path.exists(params.target_hive_interactive):
jdbc_connector(params.target_hive_interactive,
params.hive_intaractive_previous_jdbc_jar)
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}{check_db_connection_jar_name}")),
mode=0644)
File(params.start_hiveserver2_interactive_path,
mode=0755,
content=Template(format('{start_hiveserver2_interactive_script}')))
Directory(params.hive_pid_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_log_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_interactive_var_lib,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
def setup_logfeeder():
import params
Directory([
params.logfeeder_log_dir, params.logfeeder_pid_dir,
params.logfeeder_checkpoint_folder
],
mode=0755,
cd_access='a',
create_parents=True)
Directory([params.logfeeder_dir, params.logsearch_logfeeder_conf],
mode=0755,
cd_access='a',
create_parents=True,
recursive_ownership=True)
File(format("{logfeeder_log_dir}/{logfeeder_log}"), mode=0644, content='')
if params.credential_store_enabled:
params.logfeeder_env_config = update_credential_provider_path(
params.logfeeder_env_config, 'logfeeder-env',
params.logfeeder_env_jceks_file, params.logsearch_user,
params.user_group)
params.logfeeder_properties[
HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'jceks://file' + params.logfeeder_env_jceks_file
File(format("{logsearch_logfeeder_keys_folder}/ks_pass.txt"),
action="delete")
File(format("{logsearch_logfeeder_keys_folder}/ts_pass.txt"),
action="delete")
else:
Directory(params.logsearch_logfeeder_keys_folder,
cd_access='a',
mode=0755,
owner=params.logsearch_user,
group=params.user_group)
File(format("{logsearch_logfeeder_keys_folder}/ks_pass.txt"),
content=params.logfeeder_keystore_password,
mode=0600,
owner=params.logsearch_user,
group=params.user_group)
File(format("{logsearch_logfeeder_keys_folder}/ts_pass.txt"),
content=params.logfeeder_truststore_password,
mode=0600,
owner=params.logsearch_user,
group=params.user_group)
PropertiesFile(format("{logsearch_logfeeder_conf}/logfeeder.properties"),
properties=params.logfeeder_properties)
File(format("{logsearch_logfeeder_conf}/logfeeder-env.sh"),
content=InlineTemplate(params.logfeeder_env_content),
mode=0755)
File(format("{logsearch_logfeeder_conf}/log4j.xml"),
content=InlineTemplate(params.logfeeder_log4j_content))
File(format("{logsearch_logfeeder_conf}/grok-patterns"),
content=InlineTemplate(params.logfeeder_grok_patterns),
encoding="utf-8")
File(format("{logsearch_logfeeder_conf}/global.config.json"),
content=Template("global.config.json.j2"))
File(format("{logsearch_logfeeder_conf}/input.config-ambari.json"),
content=InlineTemplate(params.logfeeder_ambari_config_content),
encoding="utf-8")
File(format("{logsearch_logfeeder_conf}/output.config.json"),
content=InlineTemplate(params.logfeeder_output_config_content),
encoding="utf-8")
if params.logfeeder_system_log_enabled:
File(format(
"{logsearch_logfeeder_conf}/input.config-system_messages.json"),
content=params.logfeeder_system_messages_content)
File(format("{logsearch_logfeeder_conf}/input.config-secure_log.json"),
content=params.logfeeder_secure_log_content)
if params.logsearch_solr_kerberos_enabled:
File(format("{logfeeder_jaas_file}"),
content=Template("logfeeder_jaas.conf.j2"))
def setup_logfeeder():
import params
Directory([
params.logfeeder_log_dir, params.logfeeder_pid_dir,
params.logfeeder_checkpoint_folder
],
mode=0755,
cd_access='a',
create_parents=True)
Directory([params.logfeeder_dir, params.logsearch_logfeeder_conf],
mode=0755,
cd_access='a',
create_parents=True,
recursive_ownership=True)
File(params.logfeeder_log, mode=0644, content='')
params.logfeeder_env_config = update_credential_provider_path(
params.logfeeder_env_config, 'logfeeder-env',
params.logfeeder_env_jceks_file, params.logsearch_user,
params.user_group)
params.logfeeder_properties[
HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'jceks://file' + params.logfeeder_env_jceks_file
PropertiesFile(format("{logsearch_logfeeder_conf}/logfeeder.properties"),
properties=params.logfeeder_properties)
File(format("{logsearch_logfeeder_conf}/logfeeder-env.sh"),
content=InlineTemplate(params.logfeeder_env_content),
mode=0755)
File(format("{logsearch_logfeeder_conf}/log4j.xml"),
content=InlineTemplate(params.logfeeder_log4j_content))
File(format("{logsearch_logfeeder_conf}/grok-patterns"),
content=InlineTemplate(params.logfeeder_grok_patterns),
encoding="utf-8")
File(format("{logsearch_logfeeder_conf}/input.config-ambari.json"),
content=InlineTemplate(params.logfeeder_ambari_config_content),
encoding="utf-8")
File(format("{logsearch_logfeeder_conf}/output.config.json"),
content=InlineTemplate(params.logfeeder_output_config_content),
encoding="utf-8")
for file_name in params.logfeeder_default_config_file_names:
File(format("{logsearch_logfeeder_conf}/" + file_name),
content=Template(file_name + ".j2"))
File(format(
"{logsearch_logfeeder_conf}/input.config-logfeeder-custom.json"),
action='delete')
for service, pattern_content in params.logfeeder_metadata.iteritems():
File(format("{logsearch_logfeeder_conf}/input.config-" +
service.replace('-logsearch-conf', '') + ".json"),
content=InlineTemplate(pattern_content, extra_imports=[default]))
if params.logfeeder_system_log_enabled:
File(format(
"{logsearch_logfeeder_conf}/input.config-system_messages.json"),
content=params.logfeeder_system_messages_content)
File(format("{logsearch_logfeeder_conf}/input.config-secure_log.json"),
content=params.logfeeder_secure_log_content)
if params.logsearch_solr_kerberos_enabled:
File(format("{logfeeder_jaas_file}"),
content=Template("logfeeder_jaas.conf.j2"))
def beacon(type, action=None, upgrade_type=None):
import params
if action == 'config':
create_directory(params.beacon_home_dir)
create_directory(params.beacon_plugin_staging_dir)
cloud_cred_provider = params.beacon_cloud_cred_provider_dir.split('://')[1]
cloud_cred_parts = cloud_cred_provider.split('/', 1)
create_directory("/" + cloud_cred_parts[1], cloud_cred_parts[0])
if params.is_hive_installed:
if not isinstance(params.hive_repl_cmrootdir, UnknownConfiguration):
beacon_utils.create_hdfs_directory(params.hive_repl_cmrootdir,
params.hive_user,
01777)
if not isinstance(params.hive_repl_rootdir, UnknownConfiguration):
beacon_utils.create_hdfs_directory(params.hive_repl_rootdir,
params.hive_user,
0700)
Directory(params.beacon_pid_dir,
owner=params.beacon_user,
create_parents=True,
mode=0755,
cd_access="a",
)
Directory(params.beacon_data_dir,
owner=params.beacon_user,
create_parents=True,
mode=0755,
cd_access="a",
)
Directory(params.beacon_log_dir,
owner=params.beacon_user,
create_parents=True,
mode=0755,
cd_access="a",
)
Directory(params.beacon_webapp_dir,
owner=params.beacon_user,
create_parents=True)
Directory(params.beacon_home,
owner=params.beacon_user,
create_parents=True)
Directory(params.etc_prefix_dir,
mode=0755,
create_parents=True)
Directory(params.beacon_conf_dir,
owner=params.beacon_user,
create_parents=True)
environment_dictionary = {
"HADOOP_HOME": params.hadoop_home_dir,
"JAVA_HOME": params.java_home,
"BEACON_LOG_DIR": params.beacon_log_dir,
"BEACON_PID_DIR": params.beacon_pid_dir,
"BEACON_DATA_DIR": params.beacon_data_dir,
"BEACON_CLUSTER": params.beacon_cluster_name,
"HADOOP_CONF": params.hadoop_conf_dir
}
pid = get_user_call_output.get_user_call_output(format("cat {server_pid_file}"), user=params.beacon_user,
is_checked_call=False)[1]
process_exists = format("ls {server_pid_file} && ps -p {pid}")
if type == 'server':
if action == 'start':
try:
if params.credential_store_enabled:
if 'hadoop.security.credential.provider.path' in params.beacon_env:
credential_provider_path = params.beacon_env['hadoop.security.credential.provider.path']
credential_provider_src_path = credential_provider_path[len('jceks://file'):]
File(params.beacon_credential_provider_path[len('jceks://file'):],
owner=params.beacon_user,
group=params.user_group,
mode=0640,
content=StaticFile(credential_provider_src_path)
)
else:
Logger.error(
"hadoop.security.credential.provider.path property not found in beacon-env config-type")
File(os.path.join(params.beacon_conf_dir, 'beacon.yml'),
owner='root',
group='root',
mode=0644,
content=Template("beacon.yml.j2")
)
params.beacon_security_site = update_credential_provider_path(
params.beacon_security_site,
'beacon-security-site',
os.path.join(params.beacon_conf_dir, 'beacon-security-site.jceks'),
params.beacon_user,
params.user_group
)
XmlConfig("beacon-security-site.xml",
conf_dir=params.beacon_conf_dir,
configurations=params.beacon_security_site,
configuration_attributes=params.config['configuration_attributes']['beacon-security-site'],
owner=params.beacon_user,
group=params.user_group,
mode=0644
)
Execute(format('{beacon_home}/bin/beacon setup'),
user=params.beacon_user,
path=params.hadoop_bin_dir,
environment=environment_dictionary
)
if params.download_mysql_driver:
download_mysql_driver()
Execute(format('{beacon_home}/bin/beacon start'),
user=params.beacon_user,
path=params.hadoop_bin_dir,
environment=environment_dictionary,
not_if=process_exists,
)
if params.has_ranger_admin:
ranger_admin_url = params.config['configurations']['admin-properties']['policymgr_external_url']
ranger_admin_user = params.config['configurations']['ranger-env']['admin_username']
ranger_admin_passwd = params.config['configurations']['ranger-env']['admin_password']
if not params.security_enabled:
# Creating/Updating beacon.ranger.user with role "ROLE_SYS_ADMIN"
response_user = ranger_api_functions.get_user(ranger_admin_url, params.beacon_ranger_user,
format(
"{ranger_admin_user}:{ranger_admin_passwd}"))
if response_user is not None and response_user['name'] == params.beacon_ranger_user:
response_user_role = response_user['userRoleList'][0]
Logger.info(format(
"Beacon Ranger User with username {beacon_ranger_user} exists with role {response_user_role}"))
if response_user_role != "ROLE_SYS_ADMIN":
response_user_role = ranger_api_functions.update_user_role(ranger_admin_url,
params.beacon_ranger_user,
"ROLE_SYS_ADMIN", format(
"{ranger_admin_user}:{ranger_admin_passwd}"))
else:
response_code = ranger_api_functions.create_user(ranger_admin_url,
params.beacon_ranger_user,
params.beacon_ranger_password,
"ROLE_SYS_ADMIN", format(
"{ranger_admin_user}:{ranger_admin_passwd}"))
# Updating beacon_user role depending upon cluster environment
count = 0
while count < 10:
beacon_user_get = ranger_api_functions.get_user(ranger_admin_url, params.beacon_user, format(
"{ranger_admin_user}:{ranger_admin_passwd}"))
if beacon_user_get is not None:
break
else:
time.sleep(10) # delay for 10 seconds
count = count + 1
Logger.error(
format('Retrying to fetch {beacon_user} user from Ranger Admin for {count} time(s)'))
if beacon_user_get is not None and beacon_user_get['name'] == params.beacon_user:
beacon_user_get_role = beacon_user_get['userRoleList'][0]
if params.security_enabled and beacon_user_get_role != "ROLE_SYS_ADMIN":
beacon_service_user = ranger_api_functions.update_user_role(ranger_admin_url,
params.beacon_user,
"ROLE_SYS_ADMIN", format(
"{ranger_admin_user}:{ranger_admin_passwd}"))
elif not params.security_enabled and beacon_user_get_role != "ROLE_USER":
beacon_service_user = ranger_api_functions.update_user_role(ranger_admin_url,
params.beacon_user, "ROLE_USER",
format(
"{ranger_admin_user}:{ranger_admin_passwd}"))
if params.ranger_hive_plugin_enabled:
# Get Ranger Hive default policy for resource database, table, column
response_policy = ranger_api_functions.get_ranger_service_default_policy(ranger_admin_url,
params.service_name,
format(
"{ranger_admin_user}:{ranger_admin_passwd}"),
['database', 'table',
'column'])
if response_policy:
user_present = ranger_api_functions.check_user_policy(response_policy, params.beacon_user)
if not user_present and beacon_user_get is not None and beacon_user_get[
'name'] == params.beacon_user:
policy_id = response_policy['id']
beacon_user_policy_item = {'groups': [], 'conditions': [],
'users': [params.beacon_user],
'accesses': [{'isAllowed': True, 'type': 'all'},
{'isAllowed': True, 'type': 'repladmin'}],
'delegateAdmin': False}
policy_data = ranger_api_functions.update_policy_item(response_policy,
beacon_user_policy_item)
update_policy_response = ranger_api_functions.update_policy(ranger_admin_url, policy_id,
policy_data, format(
"{ranger_admin_user}:{ranger_admin_passwd}"))
# Get Ranger Hive default policy for resource hiveservice
response_policy = ranger_api_functions.get_ranger_service_default_policy(ranger_admin_url,
params.service_name,
format(
"{ranger_admin_user}:{ranger_admin_passwd}"),
['hiveservice'])
if response_policy:
user_present = ranger_api_functions.check_user_policy(response_policy, params.beacon_user)
if not user_present and beacon_user_get is not None and beacon_user_get[
'name'] == params.beacon_user:
# Updating beacon_user in Ranger Hive default policy for resource hiveservice
policy_id = response_policy['id']
beacon_user_policy_item = {'groups': [], 'conditions': [],
'users': [params.beacon_user],
'accesses': [{'isAllowed': True, 'type': 'serviceadmin'}],
'delegateAdmin': False}
policy_data = ranger_api_functions.update_policy_item(response_policy,
beacon_user_policy_item)
update_policy_response = ranger_api_functions.update_policy(ranger_admin_url, policy_id,
policy_data, format(
"{ranger_admin_user}:{ranger_admin_passwd}"))
if params.ranger_atlas_plugin_enabled:
# Creating beacon.atlas.user with role "ROLE_USER"
beacon_atlas_user_response = ranger_api_functions.get_user(ranger_admin_url,
params.beacon_atlas_user, format(
"{ranger_admin_user}:{ranger_admin_passwd}"))
if beacon_atlas_user_response is not None and beacon_atlas_user_response[
'name'] == params.beacon_atlas_user:
beacon_atlas_user_role = beacon_atlas_user_response['userRoleList'][0]
Logger.info(format(
"Beacon Atlas User with username {beacon_atlas_user} exists with role {beacon_atlas_user_role}"))
else:
beacon_atlas_user_create_response_code = ranger_api_functions.create_user(ranger_admin_url,
params.beacon_atlas_user,
params.beacon_atlas_password,
"ROLE_USER",
format(
"{ranger_admin_user}:{ranger_admin_passwd}"))
if params.security_enabled:
get_beacon_atlas_user = params.beacon_user
else:
get_beacon_atlas_user = params.beacon_atlas_user
if params.is_stack_3_0_or_further:
# Get Ranger Atlas default policy for ENTITY TYPE, ENTITY CLASSIFICATION and ENTITY ID resource
atlas_entity_policy_response = ranger_api_functions.get_ranger_service_default_policy(
ranger_admin_url, params.ranger_atlas_service_name,
format("{ranger_admin_user}:{ranger_admin_passwd}"),
['entity', 'entity-classification', 'entity-type'])
if atlas_entity_policy_response:
beacon_atlas_user_present = ranger_api_functions.check_user_policy(
atlas_entity_policy_response, get_beacon_atlas_user)
if not beacon_atlas_user_present:
# Updating beacon atlas user in Ranger Atlas default policy for entity resource
atlas_entity_policy_id = atlas_entity_policy_response['id']
beacon_atlas_user_policy_item = {'groups': [], 'conditions': [],
'users': [get_beacon_atlas_user], 'accesses': [
{'type': 'entity-read', 'isAllowed': True},
{'type': 'entity-create', 'isAllowed': True},
{'type': 'entity-update', 'isAllowed': True}]}
atlas_entity_policy_data = ranger_api_functions.update_policy_item(
atlas_entity_policy_response, beacon_atlas_user_policy_item)
atlas_update_entity_policy_response = ranger_api_functions.update_policy(
ranger_admin_url, atlas_entity_policy_id, atlas_entity_policy_data,
format("{ranger_admin_user}:{ranger_admin_passwd}"))
# Get Ranger Atlas default policy for ATLAS SERVICE resource
atlas_service_policy_response = ranger_api_functions.get_ranger_service_default_policy(
ranger_admin_url, params.ranger_atlas_service_name,
format("{ranger_admin_user}:{ranger_admin_passwd}"), ['atlas-service'])
if atlas_service_policy_response:
beacon_atlas_user_present = ranger_api_functions.check_user_policy(
atlas_service_policy_response, get_beacon_atlas_user)
if not beacon_atlas_user_present:
# Updating beacon atlas user in Ranger Atlas default policy for service resource
atlas_service_policy_id = atlas_service_policy_response['id']
beacon_atlas_user_policy_item = {'groups': [], 'conditions': [],
'users': [get_beacon_atlas_user], 'accesses': [
{'type': 'admin-export', 'isAllowed': True},
{'type': 'admin-import', 'isAllowed': True}]}
atlas_service_policy_data = ranger_api_functions.update_policy_item(
atlas_service_policy_response, beacon_atlas_user_policy_item)
atlas_service_policy_update_response = ranger_api_functions.update_policy(
ranger_admin_url, atlas_service_policy_id, atlas_service_policy_data,
format("{ranger_admin_user}:{ranger_admin_passwd}"))
# Get Ranger Atlas default policy for TYPE CATEGORY and TYPE resource
atlas_type_category_policy_response = ranger_api_functions.get_ranger_service_default_policy(
ranger_admin_url, params.ranger_atlas_service_name,
format("{ranger_admin_user}:{ranger_admin_passwd}"), ['type', 'type-category'])
if atlas_type_category_policy_response:
beacon_atlas_user_present = ranger_api_functions.check_user_policy(
atlas_type_category_policy_response, get_beacon_atlas_user)
if not beacon_atlas_user_present:
# Updating beacon atlas user in Ranger Atlas default policy for type category and type resource
atlas_type_category_policy_id = atlas_type_category_policy_response['id']
beacon_atlas_user_policy_item = {'groups': [], 'conditions': [],
'users': [get_beacon_atlas_user], 'accesses': [
{'type': 'type-create', 'isAllowed': True},
{'type': 'type-update', 'isAllowed': True},
{'type': 'type-delete', 'isAllowed': True}]}
atlas_type_category_policy_data = ranger_api_functions.update_policy_item(
atlas_type_category_policy_response, beacon_atlas_user_policy_item)
atlas_update_type_category_policy_response = ranger_api_functions.update_policy(
ranger_admin_url, atlas_type_category_policy_id,
atlas_type_category_policy_data,
format("{ranger_admin_user}:{ranger_admin_passwd}"))
else:
# Get Ranger Atlas default policy for ENTITY resource
atlas_policy_response = ranger_api_functions.get_ranger_service_default_policy(
ranger_admin_url, params.ranger_atlas_service_name,
format("{ranger_admin_user}:{ranger_admin_passwd}"), ['entity'])
if atlas_policy_response:
beacon_atlas_user_present = ranger_api_functions.check_user_policy(
atlas_policy_response, get_beacon_atlas_user)
if not beacon_atlas_user_present:
# Updating beacon atlas user in Ranger Atlas default policy for entity resource
atlas_policy_id = atlas_policy_response['id']
beacon_atlas_user_policy_item = {'groups': [], 'conditions': [],
'users': [get_beacon_atlas_user],
'accesses': [{'type': 'read', 'isAllowed': True},
{'type': 'create', 'isAllowed': True},
{'type': 'update', 'isAllowed': True},
{'type': 'delete', 'isAllowed': True},
{'type': 'all', 'isAllowed': True}]}
atlas_policy_data = ranger_api_functions.update_policy_item(atlas_policy_response,
beacon_atlas_user_policy_item)
atlas_update_policy_response = ranger_api_functions.update_policy(ranger_admin_url,
atlas_policy_id,
atlas_policy_data,
format(
"{ranger_admin_user}:{ranger_admin_passwd}"))
# Get Ranger Atlas default policy for OPERATION resource
atlas_operation_policy_response = ranger_api_functions.get_ranger_service_default_policy(
ranger_admin_url, params.ranger_atlas_service_name,
format("{ranger_admin_user}:{ranger_admin_passwd}"), ['operation'])
if atlas_operation_policy_response:
beacon_atlas_user_present = ranger_api_functions.check_user_policy(
atlas_operation_policy_response, get_beacon_atlas_user)
if not beacon_atlas_user_present:
# Updating beacon atlas user in Ranger Atlas default policy for operation resource
atlas_operation_policy_id = atlas_operation_policy_response['id']
beacon_atlas_user_policy_item = {'groups': [], 'conditions': [],
'users': [get_beacon_atlas_user],
'accesses': [{'type': 'read', 'isAllowed': True},
{'type': 'create', 'isAllowed': True},
{'type': 'update', 'isAllowed': True},
{'type': 'delete', 'isAllowed': True},
{'type': 'all', 'isAllowed': True}]}
atlas_operation_policy_data = ranger_api_functions.update_policy_item(
atlas_operation_policy_response, beacon_atlas_user_policy_item)
atlas_operation_policy_update_response = ranger_api_functions.update_policy(
ranger_admin_url, atlas_operation_policy_id, atlas_operation_policy_data,
format("{ranger_admin_user}:{ranger_admin_passwd}"))
except Exception as e:
show_logs(params.beacon_log_dir, params.beacon_user)
if action == 'stop':
try:
Execute(format('{beacon_home}/bin/beacon stop'),
user=params.beacon_user,
path=params.hadoop_bin_dir,
environment=environment_dictionary)
except:
show_logs(params.beacon_log_dir, params.beacon_user)
File(params.server_pid_file, action='delete')
def hive(name=None):
import params
hive_client_conf_path = format("/etc/hive/conf")
# Permissions 644 for conf dir (client) files, and 600 for conf.server
mode_identified = 0644 if params.hive_config_dir == hive_client_conf_path else 0600
if name == 'hiveserver2':
if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir:
params.HdfsResource(params.hcat_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.hcat_user,
mode=params.hcat_hdfs_user_mode)
params.HdfsResource(params.webhcat_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.webhcat_user,
mode=params.webhcat_hdfs_user_mode)
# ****** Begin Copy Tarballs ******
# *********************************
# if copy tarball to HDFS feature supported copy mapreduce.tar.gz and tez.tar.gz to HDFS
copy_to_hdfs("mapreduce",
params.user_group,
params.hdfs_user,
skip=params.sysprep_skip_copy_tarballs_hdfs)
copy_to_hdfs("tez",
params.user_group,
params.hdfs_user,
skip=params.sysprep_skip_copy_tarballs_hdfs)
# Always copy pig.tar.gz and hive.tar.gz using the appropriate mode.
# This can use a different source and dest location to account
copy_to_hdfs("pig",
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=params.pig_tar_source,
custom_dest_file=params.pig_tar_dest_file,
skip=params.sysprep_skip_copy_tarballs_hdfs)
copy_to_hdfs("hive",
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=params.hive_tar_source,
custom_dest_file=params.hive_tar_dest_file,
skip=params.sysprep_skip_copy_tarballs_hdfs)
wildcard_tarballs = ["sqoop", "hadoop_streaming"]
for tarball_name in wildcard_tarballs:
source_file_pattern = eval("params." + tarball_name +
"_tar_source")
dest_dir = eval("params." + tarball_name + "_tar_dest_dir")
if source_file_pattern is None or dest_dir is None:
continue
source_files = glob.glob(
source_file_pattern) if "*" in source_file_pattern else [
source_file_pattern
]
for source_file in source_files:
src_filename = os.path.basename(source_file)
dest_file = os.path.join(dest_dir, src_filename)
copy_to_hdfs(tarball_name,
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=source_file,
custom_dest_file=dest_file,
skip=params.sysprep_skip_copy_tarballs_hdfs)
# ******* End Copy Tarballs *******
# *********************************
# if warehouse directory is in DFS
if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(
params.default_fs).scheme:
# Create Hive Metastore Warehouse Dir
params.HdfsResource(params.hive_apps_whs_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=0777)
else:
Logger.info(
format(
"Not creating warehouse directory '{hive_apps_whs_dir}', as the location is not in DFS."
))
# Create Hive User Dir
params.HdfsResource(params.hive_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=params.hive_hdfs_user_mode)
if not is_empty(params.hive_exec_scratchdir) and not urlparse(
params.hive_exec_scratchdir).path.startswith("/tmp"):
params.HdfsResource(
params.hive_exec_scratchdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.hdfs_user,
mode=0777
) # Hive expects this dir to be writeable by everyone as it is used as a temp dir
params.HdfsResource(None, action="execute")
Directory(params.hive_etc_dir_prefix, mode=0755)
# We should change configurations for client as well as for server.
# The reason is that stale-configs are service-level, not component.
Logger.info("Directories to fill with configs: %s" %
str(params.hive_conf_dirs_list))
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
params.hive_site_config = update_credential_provider_path(
params.hive_site_config, 'hive-site',
os.path.join(params.hive_conf_dir, 'hive-site.jceks'),
params.hive_user, params.user_group)
XmlConfig(
"hive-site.xml",
conf_dir=params.hive_config_dir,
configurations=params.hive_site_config,
configuration_attributes=params.config['configuration_attributes']
['hive-site'],
owner=params.hive_user,
group=params.user_group,
mode=mode_identified)
# Generate atlas-application.properties.xml file
if params.enable_atlas_hook:
atlas_hook_filepath = os.path.join(params.hive_config_dir,
params.atlas_hook_filename)
setup_atlas_hook(SERVICE.HIVE,
params.hive_atlas_application_properties,
atlas_hook_filepath, params.hive_user,
params.user_group)
if name == 'hiveserver2':
XmlConfig(
"hiveserver2-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']['hiveserver2-site'],
configuration_attributes=params.config['configuration_attributes']
['hiveserver2-site'],
owner=params.hive_user,
group=params.user_group,
mode=0600)
if params.hive_metastore_site_supported and name == 'metastore':
XmlConfig(
"hivemetastore-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']
['hivemetastore-site'],
configuration_attributes=params.config['configuration_attributes']
['hivemetastore-site'],
owner=params.hive_user,
group=params.user_group,
mode=0600)
File(format("{hive_config_dir}/hive-env.sh"),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=InlineTemplate(params.hive_env_sh_template))
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'hive.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hive.conf.j2"))
if params.security_enabled:
File(os.path.join(params.hive_config_dir, 'zkmigrator_jaas.conf'),
owner=params.hive_user,
group=params.user_group,
content=Template("zkmigrator_jaas.conf.j2"))
if name == 'metastore' or name == 'hiveserver2':
if params.hive_jdbc_target is not None and not os.path.exists(
params.hive_jdbc_target):
jdbc_connector(params.hive_jdbc_target,
params.hive_previous_jdbc_jar)
if params.hive2_jdbc_target is not None and not os.path.exists(
params.hive2_jdbc_target):
jdbc_connector(params.hive2_jdbc_target,
params.hive2_previous_jdbc_jar)
File(
format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}{check_db_connection_jar_name}")),
mode=0644,
)
if name == 'metastore':
File(os.path.join(params.hive_server_conf_dir,
"hadoop-metrics2-hivemetastore.properties"),
owner=params.hive_user,
group=params.user_group,
mode=0600,
content=Template("hadoop-metrics2-hivemetastore.properties.j2"))
File(params.start_metastore_path,
mode=0755,
content=StaticFile('startMetastore.sh'))
elif name == 'hiveserver2':
File(params.start_hiveserver2_path,
mode=0755,
content=Template(format('{start_hiveserver2_script}')))
File(os.path.join(params.hive_server_conf_dir,
"hadoop-metrics2-hiveserver2.properties"),
owner=params.hive_user,
group=params.user_group,
mode=0600,
content=Template("hadoop-metrics2-hiveserver2.properties.j2"))
if name != "client":
Directory(params.hive_pid_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_log_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_var_lib,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
