"request {2} and direction {3}. Nothing else to do.".format(stack_name, stack_version, request_id, upgrade_direction))
# Nothing else to do here for this as it appears to have already been
# removed by another component being upgraded
return
else:
Logger.info("The marker file differs from the new value. Will proceed to delete Storm local dir, "
"and generate new file. Current marker file: {0}".format(str(existing_json_map)))
except Exception, e:
Logger.error("The marker file {0} appears to be corrupt; removing it. Error: {1}".format(marker_file, str(e)))
File(marker_file, action="delete")
else:
Logger.info('The marker file {0} does not exist; will attempt to delete local Storm directory if it exists.'.format(marker_file))
# Delete from local directory
if os.path.isdir(storm_local_directory):
Logger.info("Deleting storm local directory, {0}".format(storm_local_directory))
Directory(storm_local_directory, action="delete", recursive=True)
# Recreate storm local directory
Logger.info("Recreating storm local directory, {0}".format(storm_local_directory))
Directory(storm_local_directory, mode=0755, owner=params.storm_user,
group=params.user_group, recursive=True)
# The file doesn't exist, so create it
Logger.info("Saving marker file to {0} with contents: {1}".format(marker_file, str(json_map)))
with open(marker_file, 'w') as file_pointer:
json.dump(json_map, file_pointer, indent=2)
if __name__ == "__main__":
StormUpgrade().execute()
def setup_logsearch_solr(name=None):
import params
if name == 'server':
Directory([
params.logsearch_solr_log_dir, params.logsearch_solr_piddir,
params.logsearch_solr_datadir,
params.logsearch_solr_data_resources_dir
],
mode=0755,
cd_access='a',
create_parents=True,
owner=params.logsearch_solr_user,
group=params.user_group)
Directory([params.solr_dir, params.logsearch_solr_conf],
mode=0755,
cd_access='a',
owner=params.logsearch_solr_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True)
File(params.logsearch_solr_log,
mode=0644,
owner=params.logsearch_solr_user,
group=params.user_group,
content='')
File(format("{logsearch_solr_conf}/logsearch-solr-env.sh"),
content=InlineTemplate(params.solr_env_content),
mode=0755,
owner=params.logsearch_solr_user,
group=params.user_group)
File(format("{logsearch_solr_datadir}/solr.xml"),
content=InlineTemplate(params.solr_xml_content),
owner=params.logsearch_solr_user,
group=params.user_group)
File(format("{logsearch_solr_conf}/log4j.properties"),
content=InlineTemplate(params.solr_log4j_content),
owner=params.logsearch_solr_user,
group=params.user_group)
File(format("{logsearch_solr_datadir}/zoo.cfg"),
content=Template("zoo.cfg.j2"),
owner=params.logsearch_solr_user,
group=params.user_group)
zk_cli_prefix = format(
'export JAVA_HOME={java64_home}; {cloud_scripts}/zkcli.sh -zkhost {zookeeper_quorum}'
)
create_ambari_solr_znode(zk_cli_prefix)
url_scheme = 'https' if params.logsearch_solr_ssl_enabled else 'http'
Execute(format(
'{zk_cli_prefix}{logsearch_solr_znode} -cmd clusterprop -name urlScheme -val {url_scheme}'
),
user=params.logsearch_solr_user)
if params.security_enabled:
File(format("{logsearch_solr_jaas_file}"),
content=Template("logsearch_solr_jaas.conf.j2"),
owner=params.logsearch_solr_user)
security_content = '\'{"authentication":{"class": "org.apache.solr.security.KerberosPlugin"}}\''
else:
security_content = '\'{}\''
Execute(format(
'{zk_cli_prefix} -cmd put {logsearch_solr_znode}/security.json ') +
security_content,
user=params.logsearch_solr_user)
elif name == 'client':
solr_cloud_util.setup_solr_client(params.config)
if params.security_enabled:
File(format("{solr_client_dir}/logsearch_solr_client_jaas.conf"),
content=Template("logsearch_solr_jaas.conf.j2"),
owner=params.logsearch_solr_user)
else:
raise Fail('Nor client or server were selected to install.')
def fill_conf_dir(component_conf_dir):
import params
hive_client_conf_path = os.path.realpath(
format("{stack_root}/current/{component_directory}/conf"))
component_conf_dir = os.path.realpath(component_conf_dir)
mode_identified_for_file = 0644 if component_conf_dir == hive_client_conf_path else 0600
mode_identified_for_dir = 0755 if component_conf_dir == hive_client_conf_path else 0700
Directory(component_conf_dir,
owner=params.hive_user,
group=params.user_group,
create_parents=True,
mode=mode_identified_for_dir)
if 'mapred-site' in params.config['configurations']:
XmlConfig(
"mapred-site.xml",
conf_dir=component_conf_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configuration_attributes']
['mapred-site'],
owner=params.hive_user,
group=params.user_group,
mode=mode_identified_for_file)
File(format("{component_conf_dir}/hive-default.xml.template"),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified_for_file)
File(format("{component_conf_dir}/hive-env.sh.template"),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified_for_file)
# Create hive-log4j.properties and hive-exec-log4j.properties
# in /etc/hive/conf and not in /etc/hive2/conf
if params.log4j_version == '1':
log4j_exec_filename = 'hive-exec-log4j.properties'
if (params.log4j_exec_props != None):
File(format("{component_conf_dir}/{log4j_exec_filename}"),
mode=mode_identified_for_file,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.log4j_exec_props))
elif (os.path.exists(
"{component_conf_dir}/{log4j_exec_filename}.template")):
File(format("{component_conf_dir}/{log4j_exec_filename}"),
mode=mode_identified_for_file,
group=params.user_group,
owner=params.hive_user,
content=StaticFile(
format(
"{component_conf_dir}/{log4j_exec_filename}.template")
))
log4j_filename = 'hive-log4j.properties'
if (params.log4j_props != None):
File(format("{component_conf_dir}/{log4j_filename}"),
mode=mode_identified_for_file,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.log4j_props))
elif (os.path.exists("{component_conf_dir}/{log4j_filename}.template")
):
File(format("{component_conf_dir}/{log4j_filename}"),
mode=mode_identified_for_file,
group=params.user_group,
owner=params.hive_user,
content=StaticFile(
format("{component_conf_dir}/{log4j_filename}.template")))
if params.parquet_logging_properties is not None:
File(format("{component_conf_dir}/parquet-logging.properties"),
mode=mode_identified_for_file,
group=params.user_group,
owner=params.hive_user,
content=params.parquet_logging_properties)
def copy_jdbc_connector(stack_version=None):
import params
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source),
mode=0644)
ranger_home = params.ranger_home
if stack_version is not None:
ranger_home = format("{stack_root}/{stack_version}/ranger-admin")
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C',
params.tmp_dir),
sudo=True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive,
os.path.join(ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(ranger_home, 'ews', 'lib', 'sajdbc4.jar'), mode=0644)
Directory(params.jdbc_libs_dir, cd_access="a", create_parents=True)
Execute(as_sudo([
'yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir
],
auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
os.path.join(ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(ranger_home, 'ews', 'lib', params.jdbc_jar_name),
mode=0644)
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties=params.config['configurations']['admin-properties'],
owner=params.unix_user,
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties={
'SQL_CONNECTOR_JAR':
format('{ranger_home}/ews/lib/sajdbc4.jar')
},
owner=params.unix_user,
)
else:
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties={'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
owner=params.unix_user,
)
def setup_tagsync(upgrade_type=None):
import params
ranger_tagsync_home = params.ranger_tagsync_home
ranger_home = params.ranger_home
ranger_tagsync_conf = params.ranger_tagsync_conf
Directory(format("{ranger_tagsync_conf}"),
owner=params.unix_user,
group=params.unix_group,
create_parents=True)
Directory(
params.ranger_pid_dir,
mode=0750,
create_parents=True,
owner=params.unix_user,
group=params.unix_group,
cd_access="a",
)
Directory(params.tagsync_log_dir,
create_parents=True,
owner=params.unix_user,
group=params.unix_group,
cd_access="a",
mode=0755)
File(format('{ranger_tagsync_conf}/ranger-tagsync-env-logdir.sh'),
content=format("export RANGER_TAGSYNC_LOG_DIR={tagsync_log_dir}"),
owner=params.unix_user,
group=params.unix_group,
mode=0755)
XmlConfig(
"ranger-tagsync-site.xml",
conf_dir=ranger_tagsync_conf,
configurations=params.config['configurations']['ranger-tagsync-site'],
configuration_attributes=params.config['configuration_attributes']
['ranger-tagsync-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
PropertiesFile(
format('{ranger_tagsync_conf}/atlas-application.properties'),
properties=params.tagsync_application_properties,
mode=0755,
owner=params.unix_user,
group=params.unix_group)
File(format('{ranger_tagsync_conf}/log4j.properties'),
owner=params.unix_user,
group=params.unix_group,
content=params.tagsync_log4j,
mode=0644)
File(
params.tagsync_services_file,
mode=0755,
)
Execute(('ln', '-sf', format('{tagsync_services_file}'),
'/usr/bin/ranger-tagsync'),
not_if=format("ls /usr/bin/ranger-tagsync"),
only_if=format("ls {tagsync_services_file}"),
sudo=True)
create_core_site_xml(ranger_tagsync_conf)
def yarn(name=None, config_dir=None):
"""
:param name: Component name, apptimelineserver, nodemanager, resourcemanager, or None (defaults for client)
:param config_dir: Which config directory to write configs to, which could be different during rolling upgrade.
"""
import params
if config_dir is None:
config_dir = params.hadoop_conf_dir
if name == "historyserver":
if params.yarn_log_aggregation_enabled:
params.HdfsResource(params.yarn_nm_app_log_dir,
action="create_on_execute",
type="directory",
owner=params.yarn_user,
group=params.user_group,
mode=0777,
recursive_chmod=True)
# create the /tmp folder with proper permissions if it doesn't exist yet
if params.entity_file_history_directory.startswith('/tmp'):
params.HdfsResource(
params.hdfs_tmp_dir,
action="create_on_execute",
type="directory",
owner=params.hdfs_user,
mode=0777,
)
params.HdfsResource(params.entity_file_history_directory,
action="create_on_execute",
type="directory",
owner=params.yarn_user,
group=params.user_group)
params.HdfsResource("/mapred",
type="directory",
action="create_on_execute",
owner=params.mapred_user)
params.HdfsResource("/mapred/system",
type="directory",
action="create_on_execute",
owner=params.hdfs_user)
params.HdfsResource(params.mapreduce_jobhistory_done_dir,
type="directory",
action="create_on_execute",
owner=params.mapred_user,
group=params.user_group,
change_permissions_for_parents=True,
mode=0777)
params.HdfsResource(None, action="execute")
Directory(
params.jhs_leveldb_state_store_dir,
owner=params.mapred_user,
group=params.user_group,
create_parents=True,
cd_access="a",
recursive_ownership=True,
)
#<editor-fold desc="Node Manager Section">
if name == "nodemanager":
# First start after enabling/disabling security
if params.toggle_nm_security:
Directory(params.nm_local_dirs_list + params.nm_log_dirs_list,
action='delete')
# If yarn.nodemanager.recovery.dir exists, remove this dir
if params.yarn_nodemanager_recovery_dir:
Directory(InlineTemplate(
params.yarn_nodemanager_recovery_dir).get_content(),
action='delete')
# Setting NM marker file
if params.security_enabled:
Directory(params.nm_security_marker_dir)
File(
params.nm_security_marker,
content=
"Marker file to track first start after enabling/disabling security. "
"During first start yarn local, log dirs are removed and recreated"
)
elif not params.security_enabled:
File(params.nm_security_marker, action="delete")
if not params.security_enabled or params.toggle_nm_security:
# handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount.
nm_log_dir_to_mount_file_content = handle_mounted_dirs(
create_log_dir, params.nm_log_dirs,
params.nm_log_dir_to_mount_file, params)
# create a history file used by handle_mounted_dirs
File(params.nm_log_dir_to_mount_file,
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
content=nm_log_dir_to_mount_file_content)
nm_local_dir_to_mount_file_content = handle_mounted_dirs(
create_local_dir, params.nm_local_dirs,
params.nm_local_dir_to_mount_file, params)
File(params.nm_local_dir_to_mount_file,
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
content=nm_local_dir_to_mount_file_content)
create_hive_llap_work_dir(params)
#</editor-fold>
if params.yarn_nodemanager_recovery_dir:
Directory(
InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
mode=0755,
cd_access='a',
)
Directory(
[params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir],
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
cd_access='a',
)
Directory(
[
params.mapred_pid_dir_prefix, params.mapred_pid_dir,
params.mapred_log_dir_prefix, params.mapred_log_dir
],
owner=params.mapred_user,
group=params.user_group,
create_parents=True,
cd_access='a',
)
Directory(
[params.yarn_log_dir_prefix],
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
ignore_failures=True,
cd_access='a',
)
XmlConfig(
"core-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configuration_attributes']
['core-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644)
# During RU, Core Masters and Slaves need hdfs-site.xml
# TODO, instead of specifying individual configs, which is susceptible to breaking when new configs are added,
# RU should rely on all available in <stack-root>/<version>/hadoop/conf
if 'hdfs-site' in params.config['configurations']:
XmlConfig(
"hdfs-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configuration_attributes']
['hdfs-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644)
XmlConfig(
"mapred-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configuration_attributes']
['mapred-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
XmlConfig(
"yarn-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['yarn-site'],
configuration_attributes=params.config['configuration_attributes']
['yarn-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
XmlConfig(
"capacity-scheduler.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['capacity-scheduler'],
configuration_attributes=params.config['configuration_attributes']
['capacity-scheduler'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
if name == 'resourcemanager':
File(params.yarn_job_summary_log,
owner=params.yarn_user,
group=params.user_group)
if not is_empty(
params.node_label_enable
) and params.node_label_enable or is_empty(
params.node_label_enable) and params.node_labels_dir:
params.HdfsResource(params.node_labels_dir,
type="directory",
action="create_on_execute",
change_permissions_for_parents=True,
owner=params.yarn_user,
group=params.user_group,
mode=0700)
params.HdfsResource(None, action="execute")
elif name == 'apptimelineserver':
Directory(
params.ats_leveldb_dir,
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
cd_access="a",
)
# if stack support application timeline-service state store property (timeline_state_store stack feature)
if params.stack_supports_timeline_state_store:
Directory(
params.ats_leveldb_state_store_dir,
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
cd_access="a",
)
# app timeline server 1.5 directories
if not is_empty(params.entity_groupfs_store_dir):
parent_path = os.path.dirname(params.entity_groupfs_store_dir)
params.HdfsResource(parent_path,
type="directory",
action="create_on_execute",
change_permissions_for_parents=True,
owner=params.yarn_user,
group=params.user_group,
mode=0755)
params.HdfsResource(params.entity_groupfs_store_dir,
type="directory",
action="create_on_execute",
owner=params.yarn_user,
group=params.user_group,
mode=params.entity_groupfs_store_dir_mode)
if not is_empty(params.entity_groupfs_active_dir):
parent_path = os.path.dirname(params.entity_groupfs_active_dir)
params.HdfsResource(parent_path,
type="directory",
action="create_on_execute",
change_permissions_for_parents=True,
owner=params.yarn_user,
group=params.user_group,
mode=0755)
params.HdfsResource(params.entity_groupfs_active_dir,
type="directory",
action="create_on_execute",
owner=params.yarn_user,
group=params.user_group,
mode=params.entity_groupfs_active_dir_mode)
params.HdfsResource(None, action="execute")
File(params.rm_nodes_exclude_path,
owner=params.yarn_user,
group=params.user_group)
File(format("{limits_conf_dir}/yarn.conf"),
mode=0644,
content=Template('yarn.conf.j2'))
File(format("{limits_conf_dir}/mapreduce.conf"),
mode=0644,
content=Template('mapreduce.conf.j2'))
File(os.path.join(config_dir, "yarn-env.sh"),
owner=params.yarn_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.yarn_env_sh_template))
container_executor = format("{yarn_container_bin}/container-executor")
File(container_executor,
group=params.yarn_executor_container_group,
mode=params.container_executor_mode)
File(os.path.join(config_dir, "container-executor.cfg"),
group=params.user_group,
mode=0644,
content=Template('container-executor.cfg.j2'))
Directory(params.cgroups_dir,
group=params.user_group,
create_parents=True,
mode=0755,
cd_access="a")
if params.security_enabled:
tc_mode = 0644
tc_owner = "root"
else:
tc_mode = None
tc_owner = params.hdfs_user
File(os.path.join(config_dir, "mapred-env.sh"),
owner=tc_owner,
mode=0755,
content=InlineTemplate(params.mapred_env_sh_template))
if params.security_enabled:
File(os.path.join(params.hadoop_bin, "task-controller"),
owner="root",
group=params.mapred_tt_group,
mode=06050)
File(os.path.join(config_dir, 'taskcontroller.cfg'),
owner=tc_owner,
mode=tc_mode,
group=params.mapred_tt_group,
content=Template("taskcontroller.cfg.j2"))
else:
File(os.path.join(config_dir, 'taskcontroller.cfg'),
owner=tc_owner,
content=Template("taskcontroller.cfg.j2"))
if "mapred-site" in params.config['configurations']:
XmlConfig(
"mapred-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configuration_attributes']
['mapred-site'],
owner=params.mapred_user,
group=params.user_group)
if "capacity-scheduler" in params.config['configurations']:
XmlConfig(
"capacity-scheduler.xml",
conf_dir=config_dir,
configurations=params.config['configurations']
['capacity-scheduler'],
configuration_attributes=params.config['configuration_attributes']
['capacity-scheduler'],
owner=params.hdfs_user,
group=params.user_group)
if "ssl-client" in params.config['configurations']:
XmlConfig(
"ssl-client.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configuration_attributes']
['ssl-client'],
owner=params.hdfs_user,
group=params.user_group)
Directory(
params.hadoop_conf_secure_dir,
create_parents=True,
owner='root',
group=params.user_group,
cd_access='a',
)
XmlConfig(
"ssl-client.xml",
conf_dir=params.hadoop_conf_secure_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configuration_attributes']
['ssl-client'],
owner=params.hdfs_user,
group=params.user_group)
if "ssl-server" in params.config['configurations']:
XmlConfig(
"ssl-server.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['ssl-server'],
configuration_attributes=params.config['configuration_attributes']
['ssl-server'],
owner=params.hdfs_user,
group=params.user_group)
if os.path.exists(os.path.join(config_dir, 'fair-scheduler.xml')):
File(os.path.join(config_dir, 'fair-scheduler.xml'),
owner=params.mapred_user,
group=params.user_group)
if os.path.exists(os.path.join(config_dir, 'ssl-client.xml.example')):
File(os.path.join(config_dir, 'ssl-client.xml.example'),
owner=params.mapred_user,
group=params.user_group)
if os.path.exists(os.path.join(config_dir, 'ssl-server.xml.example')):
File(os.path.join(config_dir, 'ssl-server.xml.example'),
owner=params.mapred_user,
group=params.user_group)
def _prepare_mapreduce_tarball():
"""
Prepares the mapreduce tarball by including the native LZO libraries if necessary. If LZO is
not enabled or has not been opted-in, then this will do nothing and return the original
tarball to upload to HDFS.
:return: the full path of the newly created mapreduce tarball to use or the original path
if no changes were made
"""
# get the mapreduce tarball to crack open and add LZO libraries to
_, mapreduce_source_file, _, _ = get_tarball_paths("mapreduce")
if not lzo_utils.should_install_lzo():
return mapreduce_source_file
Logger.info("Preparing the mapreduce tarball with native LZO libraries...")
temp_dir = Script.get_tmp_dir()
# create the temp staging directories ensuring that non-root agents using tarfile can work with them
mapreduce_temp_dir = tempfile.mkdtemp(prefix="mapreduce-tarball-",
dir=temp_dir)
sudo.chmod(mapreduce_temp_dir, 0777)
# calculate the source directory for LZO
hadoop_lib_native_source_dir = os.path.join(
os.path.dirname(mapreduce_source_file), "lib", "native")
if not sudo.path_exists(hadoop_lib_native_source_dir):
raise Fail(
"Unable to seed the mapreduce tarball with native LZO libraries since the source Hadoop native lib directory {0} does not exist"
.format(hadoop_lib_native_source_dir))
Logger.info("Extracting {0} to {1}".format(mapreduce_source_file,
mapreduce_temp_dir))
tar_archive.extract_archive(mapreduce_source_file, mapreduce_temp_dir)
mapreduce_lib_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib")
# copy native libraries from source hadoop to target
Execute(("cp", "-af", hadoop_lib_native_source_dir, mapreduce_lib_dir),
sudo=True)
# ensure that the hadoop/lib/native directory is readable by non-root (which it typically is not)
Directory(mapreduce_lib_dir,
mode=0755,
cd_access='a',
recursive_ownership=True)
# create the staging directory so that non-root agents can write to it
mapreduce_native_tarball_staging_dir = os.path.join(
temp_dir, "mapreduce-native-tarball-staging")
if not os.path.exists(mapreduce_native_tarball_staging_dir):
Directory(mapreduce_native_tarball_staging_dir,
mode=0777,
cd_access='a',
create_parents=True,
recursive_ownership=True)
mapreduce_tarball_with_native_lib = os.path.join(
mapreduce_native_tarball_staging_dir, "mapreduce-native.tar.gz")
Logger.info("Creating a new mapreduce tarball at {0}".format(
mapreduce_tarball_with_native_lib))
# tar up mapreduce, making sure to specify nothing for the arcname so that it does not include an absolute path
with closing(tarfile.open(mapreduce_tarball_with_native_lib,
"w:gz")) as new_tarball:
new_tarball.add(mapreduce_temp_dir, arcname=os.path.sep)
# ensure that the tarball can be read and uploaded
sudo.chmod(mapreduce_tarball_with_native_lib, 0744)
# cleanup
sudo.rmtree(mapreduce_temp_dir)
return mapreduce_tarball_with_native_lib
def metadata(type='server'):
import params
# Needed by both Server and Client
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
if type == "server":
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(format('{conf_dir}/solr'),
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
war_source = format('{metadata_home}/server/webapp/atlas.war')
war_target = format("{expanded_war_dir}/atlas.war")
Execute(('cp', war_source, war_target),
sudo=True,
not_if=war_source == war_target)
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_log4j_content))
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content))
if not is_empty(params.atlas_admin_username) and not is_empty(
params.atlas_admin_password):
psswd_output = hashlib.sha256(
params.atlas_admin_password).hexdigest()
ModifyPropertiesFile(
format("{conf_dir}/users-credentials.properties"),
properties={
format('{atlas_admin_username}'):
format('ROLE_ADMIN::{psswd_output}')
},
owner=params.metadata_user)
files_to_chown = [
format("{conf_dir}/policy-store.txt"),
format("{conf_dir}/users-credentials.properties")
]
for file in files_to_chown:
if os.path.exists(file):
Execute(
('chown', format('{metadata_user}:{user_group}'), file),
sudo=True)
Execute(('chmod', '640', file), sudo=True)
if params.metadata_solrconfig_content:
File(format("{conf_dir}/solr/solrconfig.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_solrconfig_content))
# Needed by both Server and Client
PropertiesFile(format('{conf_dir}/{conf_file}'),
properties=params.application_properties,
mode=0600,
owner=params.metadata_user,
group=params.user_group)
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)
if type == 'server' and params.search_backend_solr and params.has_infra_solr:
solr_cloud_util.setup_solr_client(params.config)
check_znode()
jaasFile = params.atlas_jaas_file if params.security_enabled else None
upload_conf_set('atlas_configs', jaasFile)
if params.security_enabled: # update permissions before creating the collections
solr_cloud_util.add_solr_roles(
params.config,
roles=[
params.infra_solr_role_atlas,
params.infra_solr_role_ranger_audit,
params.infra_solr_role_dev
],
new_service_principals=[params.atlas_jaas_principal])
create_collection('vertex_index', 'atlas_configs', jaasFile)
create_collection('edge_index', 'atlas_configs', jaasFile)
create_collection('fulltext_index', 'atlas_configs', jaasFile)
if params.security_enabled:
secure_znode(format('{infra_solr_znode}/configs/atlas_configs'),
jaasFile)
secure_znode(format('{infra_solr_znode}/collections/vertex_index'),
jaasFile)
secure_znode(format('{infra_solr_znode}/collections/edge_index'),
jaasFile)
secure_znode(
format('{infra_solr_znode}/collections/fulltext_index'),
jaasFile)
File(params.atlas_hbase_setup,
group=params.user_group,
owner=params.hbase_user,
content=Template("atlas_hbase_setup.rb.j2"))
is_atlas_upgrade_support = check_stack_feature(
StackFeature.ATLAS_UPGRADE_SUPPORT,
get_stack_feature_version(params.config))
if is_atlas_upgrade_support and params.security_enabled:
File(params.atlas_kafka_setup,
group=params.user_group,
owner=params.kafka_user,
content=Template("atlas_kafka_acl.sh.j2"))
# files required only in case if kafka broker is not present on the host as configured component
if not params.host_with_kafka:
File(format("{kafka_conf_dir}/kafka-env.sh"),
owner=params.kafka_user,
content=InlineTemplate(params.kafka_env_sh_template))
File(format("{kafka_conf_dir}/kafka_jaas.conf"),
group=params.user_group,
owner=params.kafka_user,
content=Template("kafka_jaas.conf.j2"))
if params.stack_supports_atlas_hdfs_site_on_namenode_ha and len(
params.namenode_host) > 1:
XmlConfig(
"hdfs-site.xml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configurationAttributes']
['hdfs-site'],
owner=params.metadata_user,
group=params.user_group,
mode=0644)
else:
File(format('{conf_dir}/hdfs-site.xml'), action="delete")
'''
Atlas requires hadoop core-site.xml to resolve users/groups synced in HadoopUGI for
authentication and authorization process. Earlier the core-site.xml was available in
Hbase conf directory which is a part of Atlas class-path, from stack 2.6 onwards,
core-site.xml is no more available in Hbase conf directory. Hence need to create
core-site.xml in Atlas conf directory.
'''
if params.stack_supports_atlas_core_site and params.has_namenode:
XmlConfig(
"core-site.xml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configurationAttributes']
['core-site'],
owner=params.metadata_user,
group=params.user_group,
mode=0644)
Directory(
format('{metadata_home}/'),
owner=params.metadata_user,
group=params.user_group,
recursive_ownership=True,
)
def storm(name=None):
import params
Directory(params.log_dir,
owner=params.storm_user,
group=params.user_group,
mode=0777,
create_parents = True
)
Directory([params.pid_dir, params.local_dir],
owner=params.storm_user,
group=params.user_group,
create_parents = True,
cd_access="a",
mode=0755,
)
Directory(params.conf_dir,
group=params.user_group,
create_parents = True,
cd_access="a",
)
File(format("{conf_dir}/config.yaml"),
content=Template("config.yaml.j2"),
owner=params.storm_user,
group=params.user_group
)
configurations = params.config['configurations']['storm-site']
File(format("{conf_dir}/storm.yaml"),
content=yaml_config_template(configurations),
owner=params.storm_user,
group=params.user_group
)
File(format("{conf_dir}/storm-env.sh"),
owner=params.storm_user,
content=InlineTemplate(params.storm_env_sh_template)
)
if params.has_metric_collector:
File(format("{conf_dir}/storm-metrics2.properties"),
owner=params.storm_user,
group=params.user_group,
content=Template("storm-metrics2.properties.j2")
)
# Remove symlinks. They can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2
Link(format("{storm_lib_dir}/ambari-metrics-storm-sink.jar"),
action="delete")
# On old HDP 2.1 versions, this symlink may also exist and break EU to newer versions
Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar", action="delete")
Execute(format("{sudo} ln -s {metric_collector_sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar"),
not_if=format("ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"),
only_if=format("ls {metric_collector_sink_jar}")
)
if params.storm_logs_supported:
Directory(params.log4j_dir,
owner=params.storm_user,
group=params.user_group,
mode=0755,
create_parents = True
)
File(format("{log4j_dir}/cluster.xml"),
owner=params.storm_user,
content=InlineTemplate(params.storm_cluster_log4j_content)
)
File(format("{log4j_dir}/worker.xml"),
owner=params.storm_user,
content=InlineTemplate(params.storm_worker_log4j_content)
)
if params.security_enabled:
TemplateConfig(format("{conf_dir}/storm_jaas.conf"),
owner=params.storm_user
)
if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.2') >= 0:
TemplateConfig(format("{conf_dir}/client_jaas.conf"),
owner=params.storm_user
)
minRuid = configurations['_storm.min.ruid'] if configurations.has_key('_storm.min.ruid') else ''
min_user_ruid = int(minRuid) if minRuid.isdigit() else _find_real_user_min_uid()
File(format("{conf_dir}/worker-launcher.cfg"),
content=Template("worker-launcher.cfg.j2", min_user_ruid = min_user_ruid),
owner='root',
group=params.user_group
)
def yarn(name=None, config_dir=None):
"""
:param name: Component name, apptimelinereader, apptimelineserver, nodemanager, resourcemanager, or None (defaults for client)
:param config_dir: Which config directory to write configs to, which could be different during rolling upgrade.
"""
import params
if config_dir is None:
config_dir = params.hadoop_conf_dir
Directory(
[params.yarn_log_dir_prefix],
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
ignore_failures=True,
cd_access='a',
mode=0775,
)
Directory(
[params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir],
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
cd_access='a',
)
Directory(
[
params.mapred_pid_dir_prefix, params.mapred_pid_dir,
params.mapred_log_dir_prefix, params.mapred_log_dir
],
owner=params.mapred_user,
group=params.user_group,
create_parents=True,
cd_access='a',
)
# Some of these function calls depend on the directories above being created first.
if name == 'resourcemanager':
setup_resourcemanager()
elif name == 'nodemanager':
setup_nodemanager()
elif name == 'apptimelineserver':
setup_ats()
elif name == 'historyserver':
setup_historyserver()
generate_logfeeder_input_config(
'yarn', Template("input.config-yarn.json.j2", extra_imports=[default]))
# if there is the viewFS mount table content, create separate xml config and include in in the core-site
# else just create core-site
if params.mount_table_content:
XmlConfig(
"core-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configurationAttributes']
['core-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
xml_include_file=os.path.join(config_dir,
params.xml_inclusion_file_name))
File(os.path.join(config_dir, params.xml_inclusion_file_name),
owner=params.hdfs_user,
group=params.user_group,
content=params.mount_table_content,
mode=0644)
else:
XmlConfig(
"core-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configurationAttributes']
['core-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644)
# During RU, Core Masters and Slaves need hdfs-site.xml
XmlConfig("hdfs-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configurationAttributes']
['hdfs-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644)
XmlConfig("mapred-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configurationAttributes']
['mapred-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
configs = {}
configs.update(params.config['configurations']['yarn-site'])
configs["hadoop.registry.dns.bind-port"] = params.config['configurations'][
'yarn-env']['registry.dns.bind-port']
XmlConfig("yarn-site.xml",
conf_dir=config_dir,
configurations=configs,
configuration_attributes=params.config['configurationAttributes']
['yarn-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
XmlConfig(
"capacity-scheduler.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['capacity-scheduler'],
configuration_attributes=params.config['configurationAttributes']
['capacity-scheduler'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
XmlConfig("resource-types.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['resource-types'],
configuration_attributes=params.config['configurationAttributes']
['resource-types'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
File(format("{limits_conf_dir}/yarn.conf"),
mode=0644,
content=Template('yarn.conf.j2'))
File(format("{limits_conf_dir}/mapreduce.conf"),
mode=0644,
content=Template('mapreduce.conf.j2'))
File(os.path.join(config_dir, "yarn-env.sh"),
owner=params.yarn_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.yarn_env_sh_template))
File(format("{yarn_bin}/container-executor"),
group=params.yarn_executor_container_group,
mode=params.container_executor_mode)
File(os.path.join(config_dir, "container-executor.cfg"),
group=params.user_group,
mode=0644,
content=InlineTemplate(params.container_executor_cfg_template))
Directory(params.cgroups_dir,
group=params.user_group,
create_parents=True,
mode=0755,
cd_access="a")
File(os.path.join(config_dir, "mapred-env.sh"),
owner=params.tc_owner,
mode=0755,
content=InlineTemplate(params.mapred_env_sh_template))
if params.yarn_nodemanager_recovery_dir:
Directory(
InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
mode=0755,
cd_access='a',
)
if params.security_enabled:
File(os.path.join(params.hadoop_bin, "task-controller"),
owner="root",
group=params.mapred_tt_group,
mode=06050)
File(os.path.join(config_dir, 'taskcontroller.cfg'),
owner=params.tc_owner,
mode=params.tc_mode,
group=params.mapred_tt_group,
content=Template("taskcontroller.cfg.j2"))
File(os.path.join(config_dir, 'yarn_jaas.conf'),
owner=params.yarn_user,
group=params.user_group,
content=Template("yarn_jaas.conf.j2"))
if params.has_ats:
File(os.path.join(config_dir, 'yarn_ats_jaas.conf'),
owner=params.yarn_user,
group=params.user_group,
content=Template("yarn_ats_jaas.conf.j2"))
if params.has_registry_dns:
File(os.path.join(config_dir, 'yarn_registry_dns_jaas.conf'),
owner=params.yarn_user,
group=params.user_group,
content=Template("yarn_registry_dns_jaas.conf.j2"))
File(os.path.join(config_dir, 'yarn_nm_jaas.conf'),
owner=params.yarn_user,
group=params.user_group,
content=Template("yarn_nm_jaas.conf.j2"))
if params.has_hs:
File(os.path.join(config_dir, 'mapred_jaas.conf'),
owner=params.mapred_user,
group=params.user_group,
content=Template("mapred_jaas.conf.j2"))
else:
File(os.path.join(config_dir, 'taskcontroller.cfg'),
owner=params.tc_owner,
content=Template("taskcontroller.cfg.j2"))
XmlConfig("mapred-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configurationAttributes']
['mapred-site'],
owner=params.mapred_user,
group=params.user_group)
XmlConfig(
"capacity-scheduler.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['capacity-scheduler'],
configuration_attributes=params.config['configurationAttributes']
['capacity-scheduler'],
owner=params.hdfs_user,
group=params.user_group)
if "ssl-client" in params.config['configurations']:
XmlConfig(
"ssl-client.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configurationAttributes']
['ssl-client'],
owner=params.hdfs_user,
group=params.user_group)
Directory(
params.hadoop_conf_secure_dir,
create_parents=True,
owner='root',
group=params.user_group,
cd_access='a',
)
XmlConfig(
"ssl-client.xml",
conf_dir=params.hadoop_conf_secure_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configurationAttributes']
['ssl-client'],
owner=params.hdfs_user,
group=params.user_group)
if "ssl-server" in params.config['configurations']:
XmlConfig(
"ssl-server.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['ssl-server'],
configuration_attributes=params.config['configurationAttributes']
['ssl-server'],
owner=params.hdfs_user,
group=params.user_group)
if os.path.exists(os.path.join(config_dir, 'fair-scheduler.xml')):
File(os.path.join(config_dir, 'fair-scheduler.xml'),
owner=params.mapred_user,
group=params.user_group)
if os.path.exists(os.path.join(config_dir, 'ssl-client.xml.example')):
File(os.path.join(config_dir, 'ssl-client.xml.example'),
owner=params.mapred_user,
group=params.user_group)
if os.path.exists(os.path.join(config_dir, 'ssl-server.xml.example')):
File(os.path.join(config_dir, 'ssl-server.xml.example'),
owner=params.mapred_user,
group=params.user_group)
def setup_historyserver():
import params
if params.yarn_log_aggregation_enabled:
params.HdfsResource(params.yarn_nm_app_log_dir,
action="create_on_execute",
type="directory",
owner=params.yarn_user,
group=params.user_group,
mode=01777,
recursive_chmod=True)
# create the /tmp folder with proper permissions if it doesn't exist yet
if params.entity_file_history_directory.startswith('/tmp'):
params.HdfsResource(
params.hdfs_tmp_dir,
action="create_on_execute",
type="directory",
owner=params.hdfs_user,
mode=0777,
)
params.HdfsResource(params.yarn_system_service_dir + '/async',
action="create_on_execute",
type="directory",
owner=params.yarn_user,
group=params.user_group)
params.HdfsResource(params.yarn_system_service_dir + '/sync',
action="create_on_execute",
type="directory",
owner=params.yarn_user,
group=params.user_group)
params.HdfsResource(params.entity_file_history_directory,
action="create_on_execute",
type="directory",
owner=params.yarn_user,
group=params.user_group)
params.HdfsResource("/mapred",
type="directory",
action="create_on_execute",
owner=params.mapred_user)
params.HdfsResource("/mapred/system",
type="directory",
action="create_on_execute",
owner=params.hdfs_user)
params.HdfsResource(params.mapreduce_jobhistory_done_dir,
type="directory",
action="create_on_execute",
owner=params.mapred_user,
group=params.user_group,
change_permissions_for_parents=True,
mode=0777)
params.HdfsResource(None, action="execute")
Directory(
params.jhs_leveldb_state_store_dir,
owner=params.mapred_user,
group=params.user_group,
create_parents=True,
cd_access="a",
recursive_ownership=True,
)
generate_logfeeder_input_config(
'mapreduce2',
Template("input.config-mapreduce2.json.j2", extra_imports=[default]))
def falcon(type, action=None, upgrade_type=None):
import params
if action == 'config':
Directory(
params.falcon_pid_dir,
owner=params.falcon_user,
create_parents=True,
mode=0755,
cd_access="a",
)
Directory(
params.falcon_log_dir,
owner=params.falcon_user,
create_parents=True,
mode=0755,
cd_access="a",
)
Directory(params.falcon_webapp_dir,
owner=params.falcon_user,
create_parents=True)
Directory(params.falcon_home,
owner=params.falcon_user,
create_parents=True)
Directory(params.etc_prefix_dir, mode=0755, create_parents=True)
Directory(params.falcon_conf_dir,
owner=params.falcon_user,
create_parents=True)
File(
params.falcon_conf_dir + '/falcon-env.sh',
content=InlineTemplate(params.falcon_env_sh_template),
owner=params.falcon_user,
group=params.user_group,
)
PropertiesFile(params.falcon_conf_dir + '/client.properties',
properties=params.falcon_client_properties,
mode=0644,
owner=params.falcon_user)
PropertiesFile(params.falcon_conf_dir + '/runtime.properties',
properties=params.falcon_runtime_properties,
mode=0644,
owner=params.falcon_user)
PropertiesFile(params.falcon_conf_dir + '/startup.properties',
properties=params.falcon_startup_properties,
mode=0644,
owner=params.falcon_user)
File(params.falcon_conf_dir + '/log4j.properties',
content=InlineTemplate(params.falcon_log4j),
group=params.user_group,
mode=0644,
owner=params.falcon_user)
if params.falcon_graph_storage_directory:
Directory(params.falcon_graph_storage_directory,
owner=params.falcon_user,
group=params.user_group,
mode=0775,
create_parents=True,
cd_access="a")
if params.falcon_graph_serialize_path:
Directory(params.falcon_graph_serialize_path,
owner=params.falcon_user,
group=params.user_group,
mode=0775,
create_parents=True,
cd_access="a")
# Generate atlas-application.properties.xml file
if params.falcon_atlas_support and params.enable_atlas_hook:
# If Atlas is added later than Falcon, this package will be absent.
if check_stack_feature(
StackFeature.ATLAS_INSTALL_HOOK_PACKAGE_SUPPORT,
params.current_version_formatted):
install_atlas_hook_packages(
params.atlas_plugin_package,
params.atlas_ubuntu_plugin_package,
params.host_sys_prepped,
params.agent_stack_retry_on_unavailability,
params.agent_stack_retry_count)
atlas_hook_filepath = os.path.join(params.falcon_conf_dir,
params.atlas_hook_filename)
setup_atlas_hook(SERVICE.FALCON,
params.falcon_atlas_application_properties,
atlas_hook_filepath, params.falcon_user,
params.user_group)
# Falcon 0.10 uses FALCON_EXTRA_CLASS_PATH.
# Setup symlinks for older versions.
if params.current_version_formatted and check_stack_feature(
StackFeature.FALCON_ATLAS_SUPPORT_2_3,
params.current_version_formatted):
setup_atlas_jar_symlinks("falcon", params.falcon_webinf_lib)
if type == 'server':
if action == 'config':
if params.store_uri[0:4] == "hdfs":
params.HdfsResource(params.store_uri,
type="directory",
action="create_on_execute",
owner=params.falcon_user,
mode=0755)
elif params.store_uri[0:4] == "file":
Directory(params.store_uri[7:],
owner=params.falcon_user,
create_parents=True)
# TODO change to proper mode
params.HdfsResource(params.falcon_apps_dir,
type="directory",
action="create_on_execute",
owner=params.falcon_user,
mode=0777)
# In HDP 2.4 and earlier, the data-mirroring directory was copied to HDFS.
if params.supports_data_mirroring:
params.HdfsResource(params.dfs_data_mirroring_dir,
type="directory",
action="create_on_execute",
owner=params.falcon_user,
group=params.proxyuser_group,
recursive_chown=True,
recursive_chmod=True,
mode=0770,
source=params.local_data_mirroring_dir)
# Falcon Extensions were supported in HDP 2.5 and higher.
effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(
params.version)
supports_falcon_extensions = effective_version and check_stack_feature(
StackFeature.FALCON_EXTENSIONS, effective_version)
if supports_falcon_extensions:
params.HdfsResource(params.falcon_extensions_dest_dir,
type="directory",
action="create_on_execute",
owner=params.falcon_user,
group=params.proxyuser_group,
recursive_chown=True,
recursive_chmod=True,
mode=0755,
source=params.falcon_extensions_source_dir)
# Create the extensons HiveDR store
params.HdfsResource(os.path.join(
params.falcon_extensions_dest_dir, "mirroring"),
type="directory",
action="create_on_execute",
owner=params.falcon_user,
group=params.proxyuser_group,
mode=0770)
# At least one HDFS Dir should be created, so execute the change now.
params.HdfsResource(None, action="execute")
Directory(params.falcon_local_dir,
owner=params.falcon_user,
create_parents=True,
cd_access="a")
if params.falcon_embeddedmq_enabled == True:
Directory(os.path.abspath(
os.path.join(params.falcon_embeddedmq_data, "..")),
owner=params.falcon_user,
create_parents=True)
Directory(params.falcon_embeddedmq_data,
owner=params.falcon_user,
create_parents=True)
# although Falcon's falcon-config.sh will use 'which hadoop' to figure
# this out, in an upgraded cluster, it's possible that 'which hadoop'
# still points to older binaries; it's safer to just pass in the
# hadoop home directory to use
environment_dictionary = {"HADOOP_HOME": params.hadoop_home_dir}
pid = get_user_call_output.get_user_call_output(
format("cat {server_pid_file}"),
user=params.falcon_user,
is_checked_call=False)[1]
process_exists = format("ls {server_pid_file} && ps -p {pid}")
if action == 'start':
try:
Execute(
format('{falcon_home}/bin/falcon-config.sh server falcon'),
user=params.falcon_user,
path=params.hadoop_bin_dir,
environment=environment_dictionary,
not_if=process_exists,
)
except:
show_logs(params.falcon_log_dir, params.falcon_user)
raise
if not os.path.exists(params.target_jar_file):
try:
File(params.target_jar_file,
content=DownloadSource(params.bdb_resource_name),
mode=0755)
except:
exc_msg = traceback.format_exc()
exception_message = format(
"Caught Exception while downloading {bdb_resource_name}:\n{exc_msg}"
)
Logger.error(exception_message)
if not os.path.isfile(params.target_jar_file):
error_message = """
If you are using bdb as the Falcon graph db store, please run
ambari-server setup --jdbc-db=bdb --jdbc-driver=<path to je5.0.73.jar>
on the ambari server host. Otherwise falcon startup will fail.
Otherwise please configure Falcon to use HBase as the backend as described
in the Falcon documentation.
"""
Logger.error(error_message)
try:
Execute(
format(
'{falcon_home}/bin/falcon-start -port {falcon_port}'),
user=params.falcon_user,
path=params.hadoop_bin_dir,
environment=environment_dictionary,
not_if=process_exists,
)
except:
show_logs(params.falcon_log_dir, params.falcon_user)
raise
if action == 'stop':
try:
Execute(format('{falcon_home}/bin/falcon-stop'),
user=params.falcon_user,
path=params.hadoop_bin_dir,
environment=environment_dictionary)
except:
show_logs(params.falcon_log_dir, params.falcon_user)
raise
File(params.server_pid_file, action='delete')
def prepare_libext_directory(upgrade_type=None):
"""
Performs the following actions on libext:
- creates <stack-root>/current/oozie/libext and recursively
- set 777 permissions on it and its parents.
- downloads JDBC driver JAR if needed
- copies Falcon JAR for the Oozie WAR if needed
"""
import params
# some stack versions don't need the lzo compression libraries
target_version_needs_compression_libraries = check_stack_feature(StackFeature.LZO,
params.version_for_stack_feature_checks)
# ensure the directory exists
Directory(params.oozie_libext_dir, mode = 0777)
# get all hadooplzo* JAR files
# <stack-selector-tool> set hadoop-client has not run yet, therefore we cannot use
# <stack-root>/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 2.2.0.0; don't do this
# for downgrade to 2.2.0.0 since hadoop-lzo will not be present
# This can also be called during a Downgrade.
# When a version is Installed, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries):
# ensure that the LZO files are installed for this version of Oozie
lzo_utils.install_lzo_if_needed()
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format("{stack_root}/{version}/hadoop/lib")
files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
if not files:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy files into libext
files_copied = False
for file in files:
if os.path.isfile(file):
Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_dir))
shutil.copy2(file, params.oozie_libext_dir)
files_copied = True
if not files_copied:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# ExtJS is used to build a working Oozie Web UI - without it, Oozie will startup and work
# but will not have a functioning user interface - Some stacks no longer ship ExtJS,
# so it's optional now. On an upgrade, we should make sure that if it's not found, that's OK
# However, if it is found on the system (from an earlier install) then it should be used
extjs_included = check_stack_feature(StackFeature.OOZIE_EXTJS_INCLUDED, params.version_for_stack_feature_checks)
# something like <stack-root>/current/oozie-server/libext/ext-2.2.zip
oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, params.ext_js_file)
# Copy ext ZIP to libext dir
# Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
source_ext_zip_paths = oozie.get_oozie_ext_zip_source_paths(upgrade_type, params)
found_at_least_one_oozie_ext_file = False
# Copy the first oozie ext-2.2.zip file that is found.
# This uses a list to handle the cases when migrating from some versions of BigInsights to HDP.
if source_ext_zip_paths is not None:
for source_ext_zip_path in source_ext_zip_paths:
if os.path.isfile(source_ext_zip_path):
found_at_least_one_oozie_ext_file = True
Logger.info("Copying {0} to {1}".format(source_ext_zip_path, params.oozie_libext_dir))
Execute(("cp", source_ext_zip_path, params.oozie_libext_dir), sudo=True)
Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True)
File(oozie_ext_zip_target_path, mode=0644)
break
# ExtJS was expected to the be on the system, but was not found
if extjs_included and not found_at_least_one_oozie_ext_file:
raise Fail("Unable to find any Oozie source extension files from the following paths {0}".format(source_ext_zip_paths))
# ExtJS is not expected, so it's OK - just log a warning
if not found_at_least_one_oozie_ext_file:
Logger.warning("Unable to find ExtJS in any of the following paths. The Oozie UI will not be available. Source Paths: {0}".format(source_ext_zip_paths))
# Redownload jdbc driver to a new current location
oozie.download_database_library_if_needed()
# get the upgrade version in the event that it's needed
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None:
raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.")
stack_version = upgrade_stack[1]
# copy the Falcon JAR if needed; falcon has not upgraded yet, so we must
# use the versioned falcon directory
if params.has_falcon_host:
versioned_falcon_jar_directory = "{0}/{1}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(params.stack_root, stack_version)
Logger.info("Copying {0} to {1}".format(versioned_falcon_jar_directory, params.oozie_libext_dir))
Execute(format('{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}'))
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'))
def zookeeper(type=None, upgrade_type=None):
import params
# Directory(params.config_dir,
# owner=params.zk_user,
# create_parents = True,
# group=params.user_group
# )
File(os.path.join(params.config_dir, "zookeeper-env.sh"),
content=InlineTemplate(params.zk_env_sh_template),
owner=params.zk_user,
group=params.user_group)
configFile("zoo.cfg", template_name="zoo.cfg.j2")
configFile("configuration.xsl", template_name="configuration.xsl.j2")
Directory(
params.zk_pid_dir,
owner=params.zk_user,
create_parents=True,
group=params.user_group,
mode=0755,
)
Directory(
params.zk_log_dir,
owner=params.zk_user,
create_parents=True,
group=params.user_group,
mode=0755,
)
Directory(
params.zk_data_dir,
owner=params.zk_user,
create_parents=True,
cd_access="a",
group=params.user_group,
mode=0755,
)
if type == 'server':
myid = str(sorted(params.zookeeper_hosts).index(params.hostname) + 1)
File(os.path.join(params.zk_data_dir, "myid"), mode=0644, content=myid)
generate_logfeeder_input_config(
'zookeeper',
Template("input.config-zookeeper.json.j2",
extra_imports=[default]))
if (params.log4j_props != None):
File(os.path.join(params.config_dir, "log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.zk_user,
content=InlineTemplate(params.log4j_props))
elif (os.path.exists(os.path.join(params.config_dir, "log4j.properties"))):
File(os.path.join(params.config_dir, "log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.zk_user)
if params.security_enabled:
if type == "server":
configFile("zookeeper_jaas.conf",
template_name="zookeeper_jaas.conf.j2")
configFile("zookeeper_client_jaas.conf",
template_name="zookeeper_client_jaas.conf.j2")
else:
configFile("zookeeper_client_jaas.conf",
template_name="zookeeper_client_jaas.conf.j2")
File(os.path.join(params.config_dir, "zoo_sample.cfg"),
owner=params.zk_user,
group=params.user_group)
def hive_interactive(name=None):
import params
# list of properties that should be excluded from the config
# this approach is a compromise against adding a dedicated config
# type for hive_server_interactive or needed config groups on a
# per component basis
exclude_list = ['hive.enforce.bucketing', 'hive.enforce.sorting']
# Copy Tarballs in HDFS.
if params.stack_version_formatted_major and check_stack_feature(
StackFeature.ROLLING_UPGRADE,
params.stack_version_formatted_major):
resource_created = copy_to_hdfs(
"tez_hive2",
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
host_sys_prepped=params.host_sys_prepped)
if resource_created:
params.HdfsResource(None, action="execute")
Directory(params.hive_interactive_etc_dir_prefix, mode=0755)
Logger.info("Directories to fill with configs: %s" %
str(params.hive_conf_dirs_list))
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
'''
As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml,
we need to merge hive/hive-site.xml and hive2/hive-site.xml and store it in hive2/hive-site.xml.
'''
merged_hive_interactive_site = {}
merged_hive_interactive_site.update(
params.config['configurations']['hive-site'])
merged_hive_interactive_site.update(
params.config['configurations']['hive-interactive-site'])
for item in exclude_list:
if item in merged_hive_interactive_site.keys():
del merged_hive_interactive_site[item]
'''
Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook',
which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed.
'''
remove_atlas_hook_if_exists(merged_hive_interactive_site)
'''
As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml,
we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml.
'''
merged_tez_interactive_site = {}
if 'tez-site' in params.config['configurations']:
merged_tez_interactive_site.update(
params.config['configurations']['tez-site'])
Logger.info(
"Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'."
)
else:
Logger.error(
"Tez's 'tez-site' couldn't be retrieved from passed-in configurations."
)
merged_tez_interactive_site.update(
params.config['configurations']['tez-interactive-site'])
XmlConfig(
"tez-site.xml",
conf_dir=params.tez_interactive_config_dir,
configurations=merged_tez_interactive_site,
configuration_attributes=params.config['configuration_attributes']
['tez-interactive-site'],
owner=params.tez_interactive_user,
group=params.user_group,
mode=0664)
# Create config files under /etc/hive2/conf and /etc/hive2/conf/conf.server:
# hive-site.xml
# hive-env.sh
# llap-daemon-log4j2.properties
# llap-cli-log4j2.properties
# hive-log4j2.properties
# hive-exec-log4j2.properties
# beeline-log4j2.properties
for conf_dir in params.hive_conf_dirs_list:
XmlConfig(
"hive-site.xml",
conf_dir=conf_dir,
configurations=merged_hive_interactive_site,
configuration_attributes=params.config['configuration_attributes']
['hive-interactive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
hive_server_interactive_conf_dir = conf_dir
File(format("{hive_server_interactive_conf_dir}/hive-env.sh"),
owner=params.hive_user,
group=params.user_group,
content=InlineTemplate(params.hive_interactive_env_sh_template))
llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"),
mode=0644,
group=params.user_group,
owner=params.hive_user,
content=params.llap_daemon_log4j)
llap_cli_log4j2_filename = 'llap-cli-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"),
mode=0644,
group=params.user_group,
owner=params.hive_user,
content=params.llap_cli_log4j2)
hive_log4j2_filename = 'hive-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"),
mode=0644,
group=params.user_group,
owner=params.hive_user,
content=params.hive_log4j2)
hive_exec_log4j2_filename = 'hive-exec-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"),
mode=0644,
group=params.user_group,
owner=params.hive_user,
content=params.hive_exec_log4j2)
beeline_log4j2_filename = 'beeline-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{beeline_log4j2_filename}"),
mode=0644,
group=params.user_group,
owner=params.hive_user,
content=params.beeline_log4j2)
File(format(
"{hive_server_interactive_conf_dir}/hadoop-metrics2-llapdaemon.properties"
),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-llapdaemon.j2"))
File(format(
"{hive_server_interactive_conf_dir}/hadoop-metrics2-llaptaskscheduler.properties"
),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-llaptaskscheduler.j2"))
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'hive.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hive.conf.j2"))
if not os.path.exists(params.target_hive_interactive):
jdbc_connector(params.target_hive_interactive,
params.hive_intaractive_previous_jdbc_jar)
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}{check_db_connection_jar_name}")),
mode=0644)
File(params.start_hiveserver2_interactive_path,
mode=0755,
content=Template(format('{start_hiveserver2_interactive_script}')))
Directory(params.hive_pid_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_log_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_interactive_var_lib,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
def upgrade_schema(self, env):
"""
Executes the schema upgrade binary. This is its own function because it could
be called as a standalone task from the upgrade pack, but is safe to run it for each
metastore instance. The schema upgrade on an already upgraded metastore is a NOOP.
The metastore schema upgrade requires a database driver library for most
databases. During an upgrade, it's possible that the library is not present,
so this will also attempt to copy/download the appropriate driver.
"""
Logger.info("Upgrading Hive Metastore")
import params
env.set_params(params)
if params.security_enabled:
kinit_command=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}; ")
Execute(kinit_command,user=params.smokeuser)
# ensure that the JDBC drive is present for the schema tool; if it's not
# present, then download it first
if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
target_directory = format("/usr/hdp/{version}/hive/lib")
# download it if it does not exist
if not os.path.exists(params.source_jdbc_file):
jdbc_connector()
target_directory_and_filename = os.path.join(target_directory, os.path.basename(params.source_jdbc_file))
if params.sqla_db_used:
target_native_libs_directory = format("{target_directory}/native/lib64")
Execute(format("yes | {sudo} cp {jars_in_hive_lib} {target_directory}"))
Directory(target_native_libs_directory, recursive=True)
Execute(format("yes | {sudo} cp {libs_in_hive_lib} {target_native_libs_directory}"))
Execute(format("{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*"))
else:
# copy the JDBC driver from the older metastore location to the new location only
# if it does not already exist
if not os.path.exists(target_directory_and_filename):
Execute(('cp', params.source_jdbc_file, target_directory),
path=["/bin", "/usr/bin/"], sudo = True)
File(target_directory_and_filename, mode = 0644)
# build the schema tool command
binary = format("/usr/hdp/{version}/hive/bin/schematool")
# the conf.server directory changed locations between HDP 2.2 and 2.3
# since the configurations have not been written out yet during an upgrade
# we need to choose the original legacy location
schematool_hive_server_conf_dir = params.hive_server_conf_dir
if params.current_version is not None:
current_version = format_hdp_stack_version(params.current_version)
if compare_versions(current_version, "2.3") < 0:
schematool_hive_server_conf_dir = LEGACY_HIVE_SERVER_CONF
env_dict = {
'HIVE_CONF_DIR': schematool_hive_server_conf_dir
}
command = format("{binary} -dbType {hive_metastore_db_type} -upgradeSchema")
Execute(command, user=params.hive_user, tries=1, environment=env_dict, logoutput=True)
def hive(name=None):
import params
hive_client_conf_path = format(
"{stack_root}/current/{component_directory}/conf")
# Permissions 644 for conf dir (client) files, and 600 for conf.server
mode_identified = 0644 if params.hive_config_dir == hive_client_conf_path else 0600
Directory(params.hive_etc_dir_prefix, mode=0755)
# We should change configurations for client as well as for server.
# The reason is that stale-configs are service-level, not component.
Logger.info("Directories to fill with configs: %s" %
str(params.hive_conf_dirs_list))
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
XmlConfig(
"hive-site.xml",
conf_dir=params.hive_config_dir,
configurations=params.hive_site_config,
configuration_attributes=params.config['configuration_attributes']
['hive-site'],
owner=params.hive_user,
group=params.user_group,
mode=mode_identified)
# Generate atlas-application.properties.xml file
if params.enable_atlas_hook:
atlas_hook_filepath = os.path.join(params.hive_config_dir,
params.atlas_hook_filename)
setup_atlas_hook(SERVICE.HIVE,
params.hive_atlas_application_properties,
atlas_hook_filepath, params.hive_user,
params.user_group)
File(format("{hive_config_dir}/hive-env.sh"),
owner=params.hive_user,
group=params.user_group,
content=InlineTemplate(params.hive_env_sh_template),
mode=mode_identified)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'hive.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hive.conf.j2"))
File(
format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}{check_db_connection_jar_name}")),
mode=0644,
)
if name != "client":
setup_non_client()
if name == 'hiveserver2':
setup_hiveserver2()
if name == 'metastore':
setup_metastore() # schematool work
def metadata(type='server'):
import params
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
File(format("{expanded_war_dir}/atlas.war"),
content = StaticFile(format('{metadata_home}/server/webapp/atlas.war'))
)
PropertiesFile(format('{conf_dir}/{conf_file}'),
properties = params.application_properties,
mode=0644,
owner=params.metadata_user,
group=params.user_group
)
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content)
)
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_log4j_content)
)
File(format("{conf_dir}/users-credentials.properties"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=StaticFile('users-credentials.properties')
)
File(format("{conf_dir}/policy-store.txt"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=StaticFile('policy-store.txt')
)
if type == 'server':
random_num = random.random()
upload_conf_set('basic_configs', random_num)
create_collection('vertex_index', 'basic_configs')
create_collection('edge_index', 'basic_configs')
create_collection('fulltext_index', 'basic_configs')
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)
def namenode(action=None, do_format=True, upgrade_type=None, env=None):
import params
#we need this directory to be present before any action(HA manual steps for
#additional namenode)
if action == "configure":
create_name_dirs(params.dfs_name_dir)
if action == "start":
if do_format:
format_namenode()
pass
File(params.exclude_file_path,
content=Template("exclude_hosts_list.j2"),
owner=params.hdfs_user,
group=params.user_group
)
Directory(params.hadoop_pid_dir_prefix,
mode=0755,
owner=params.hdfs_user,
group=params.user_group
)
if params.dfs_ha_enabled and \
params.dfs_ha_namenode_standby is not None and \
params.hostname == params.dfs_ha_namenode_standby:
# if the current host is the standby NameNode in an HA deployment
# run the bootstrap command, to start the NameNode in standby mode
# this requires that the active NameNode is already up and running,
# so this execute should be re-tried upon failure, up to a timeout
success = bootstrap_standby_namenode(params)
if not success:
raise Fail("Could not bootstrap standby namenode")
if upgrade_type == "rolling":
# Must start Zookeeper Failover Controller if it exists on this host because it could have been killed in order to initiate the failover.
safe_zkfc_op(action, env)
#options = "-rollingUpgrade started" if rolling_restart else ""
options = ""
if upgrade_type == "rolling":
if params.upgrade_direction == Direction.UPGRADE:
options = "-rollingUpgrade started"
elif params.upgrade_direction == Direction.DOWNGRADE:
options = "-rollingUpgrade downgrade"
elif upgrade_type == "nonrolling":
is_previous_image_dir = is_previous_fs_image()
Logger.info(format("Previous file system image dir present is {is_previous_image_dir}"))
if params.upgrade_direction == Direction.UPGRADE:
options = "-rollingUpgrade started"
elif params.upgrade_direction == Direction.DOWNGRADE:
options = "-rollingUpgrade downgrade"
Logger.info(format("Option for start command: {options}"))
service(
action="start",
name="namenode",
user=params.hdfs_user,
options=options,
create_pid_dir=True,
create_log_dir=True
)
if params.security_enabled:
Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"),
user = params.hdfs_user)
if params.dfs_ha_enabled:
is_active_namenode_cmd = as_user(format("hdfs --config {hadoop_conf_dir} haadmin -getServiceState {namenode_id} | grep active"), params.hdfs_user, env={'PATH':params.hadoop_bin_dir})
else:
is_active_namenode_cmd = True
# During NonRolling Upgrade, both NameNodes are initially down,
# so no point in checking if this is the active or standby.
if upgrade_type == "nonrolling":
is_active_namenode_cmd = False
# ___Scenario___________|_Expected safemode state__|_Wait for safemode OFF____|
# no-HA | ON -> OFF | Yes |
# HA and active | ON -> OFF | Yes |
# HA and standby | no change | no check |
# RU with HA on active | ON -> OFF | Yes |
# RU with HA on standby | ON -> OFF | Yes |
# EU with HA on active | no change | no check |
# EU with HA on standby | no change | no check |
# EU non-HA | no change | no check |
check_for_safemode_off = False
msg = ""
if params.dfs_ha_enabled:
if upgrade_type is not None:
check_for_safemode_off = True
msg = "Must wait to leave safemode since High Availability is enabled during a Stack Upgrade"
else:
Logger.info("Wait for NameNode to become active.")
if is_active_namenode(): # active
check_for_safemode_off = True
msg = "Must wait to leave safemode since High Availability is enabled and this is the Active NameNode."
else:
msg = "Will remain in the current safemode state."
else:
msg = "Must wait to leave safemode since High Availability is not enabled."
check_for_safemode_off = True
Logger.info(msg)
# During a NonRolling (aka Express Upgrade), stay in safemode since the DataNodes are down.
stay_in_safe_mode = False
if upgrade_type == "nonrolling":
stay_in_safe_mode = True
if check_for_safemode_off:
Logger.info("Stay in safe mode: {0}".format(stay_in_safe_mode))
if not stay_in_safe_mode:
wait_for_safemode_off()
# Always run this on non-HA, or active NameNode during HA.
create_hdfs_directories(is_active_namenode_cmd)
'''if params.dfs_ha_enabled:
dfs_check_nn_status_cmd = as_user(format("hdfs --config {hadoop_conf_dir} haadmin -getServiceState {namenode_id} | grep active"), params.hdfs_user, env={'PATH':params.hadoop_bin_dir})
else:
dfs_check_nn_status_cmd = None
namenode_safe_mode_off = format("hdfs dfsadmin -fs {namenode_address} -safemode get | grep 'Safe mode is OFF'")
# If HA is enabled and it is in standby, then stay in safemode, otherwise, leave safemode.
leave_safe_mode = True
if dfs_check_nn_status_cmd is not None:
code, out = shell.call(dfs_check_nn_status_cmd) # If active NN, code will be 0
if code != 0:
leave_safe_mode = False
if leave_safe_mode:
# First check if Namenode is not in 'safemode OFF' (equivalent to safemode ON), if so, then leave it
code, out = shell.call(namenode_safe_mode_off)
if code != 0:
leave_safe_mode_cmd = format("hdfs --config {hadoop_conf_dir} dfsadmin -fs {namenode_address} -safemode leave")
Execute(leave_safe_mode_cmd,
tries=10,
try_sleep=10,
user=params.hdfs_user,
path=[params.hadoop_bin_dir],
)
# Verify if Namenode should be in safemode OFF
Execute(namenode_safe_mode_off,
tries=40,
try_sleep=10,
path=[params.hadoop_bin_dir],
user=params.hdfs_user,
only_if=dfs_check_nn_status_cmd #skip when HA not active
)
create_hdfs_directories(dfs_check_nn_status_cmd)'''
if action == "stop":
service(
action="stop", name="namenode",
user=params.hdfs_user
)
if action == "decommission":
decommission()
def kafka(upgrade_type=None):
import params
ensure_base_directories()
kafka_server_config = mutable_config_dict(params.config['configurations']['kafka-broker'])
# This still has an issue of hostnames being alphabetically out-of-order for broker.id in HDP-2.2.
# Starting in HDP 2.3, Kafka handles the generation of broker.id so Ambari doesn't have to.
effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(params.version)
Logger.info(format("Effective stack version: {effective_version}"))
# In HDP-2.2 (Apache Kafka 0.8.1.1) we used to generate broker.ids based on hosts and add them to
# kafka's server.properties. In future version brokers can generate their own ids based on zookeeper seq
# We need to preserve the broker.id when user is upgrading from HDP-2.2 to any higher version.
# Once its preserved it will be written to kafka.log.dirs/meta.properties and it will be used from there on
# similarly we need preserve port as well during the upgrade
if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and \
check_stack_feature(StackFeature.CREATE_KAFKA_BROKER_ID, params.current_version) and \
check_stack_feature(StackFeature.KAFKA_LISTENERS, params.version):
if len(params.kafka_hosts) > 0 and params.hostname in params.kafka_hosts:
brokerid = str(sorted(params.kafka_hosts).index(params.hostname))
kafka_server_config['broker.id'] = brokerid
Logger.info(format("Calculating broker.id as {brokerid}"))
if 'port' in kafka_server_config:
port = kafka_server_config['port']
Logger.info(format("Port config from previous verson: {port}"))
listeners = kafka_server_config['listeners']
kafka_server_config['listeners'] = listeners.replace("6667", port)
Logger.info(format("Kafka listeners after the port update: {listeners}"))
del kafka_server_config['port']
if effective_version is not None and effective_version != "" and \
check_stack_feature(StackFeature.CREATE_KAFKA_BROKER_ID, effective_version):
if len(params.kafka_hosts) > 0 and params.hostname in params.kafka_hosts:
brokerid = str(sorted(params.kafka_hosts).index(params.hostname))
kafka_server_config['broker.id'] = brokerid
Logger.info(format("Calculating broker.id as {brokerid}"))
# listeners and advertised.listeners are only added in 2.3.0.0 onwards.
if effective_version is not None and effective_version != "" and \
check_stack_feature(StackFeature.KAFKA_LISTENERS, effective_version):
listeners = kafka_server_config['listeners'].replace("localhost", params.hostname)
Logger.info(format("Kafka listeners: {listeners}"))
if params.security_enabled and params.kafka_kerberos_enabled:
Logger.info("Kafka kerberos security is enabled.")
if "SASL" not in listeners:
listeners = listeners.replace("PLAINTEXT", "PLAINTEXTSASL")
kafka_server_config['listeners'] = listeners
kafka_server_config['advertised.listeners'] = listeners
Logger.info(format("Kafka advertised listeners: {listeners}"))
else:
kafka_server_config['listeners'] = listeners
if 'advertised.listeners' in kafka_server_config:
advertised_listeners = kafka_server_config['advertised.listeners'].replace("localhost", params.hostname)
kafka_server_config['advertised.listeners'] = advertised_listeners
Logger.info(format("Kafka advertised listeners: {advertised_listeners}"))
else:
kafka_server_config['host.name'] = params.hostname
if params.has_metric_collector:
kafka_server_config['kafka.timeline.metrics.host'] = params.metric_collector_host
kafka_server_config['kafka.timeline.metrics.port'] = params.metric_collector_port
kafka_server_config['kafka.timeline.metrics.protocol'] = params.metric_collector_protocol
kafka_server_config['kafka.timeline.metrics.truststore.path'] = params.metric_truststore_path
kafka_server_config['kafka.timeline.metrics.truststore.type'] = params.metric_truststore_type
kafka_server_config['kafka.timeline.metrics.truststore.password'] = params.metric_truststore_password
kafka_data_dir = kafka_server_config['log.dirs']
kafka_data_dirs = filter(None, kafka_data_dir.split(","))
Directory(kafka_data_dirs,
mode=0755,
cd_access='a',
owner=params.kafka_user,
group=params.user_group,
create_parents = True,
recursive_ownership = True,
)
PropertiesFile("server.properties",
dir=params.conf_dir,
properties=kafka_server_config,
owner=params.kafka_user,
group=params.user_group,
)
File(format("{conf_dir}/kafka-env.sh"),
owner=params.kafka_user,
content=InlineTemplate(params.kafka_env_sh_template)
)
if (params.log4j_props != None):
File(format("{conf_dir}/log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.kafka_user,
content=params.log4j_props
)
if params.security_enabled and params.kafka_kerberos_enabled:
TemplateConfig(format("{conf_dir}/kafka_jaas.conf"),
owner=params.kafka_user)
TemplateConfig(format("{conf_dir}/kafka_client_jaas.conf"),
owner=params.kafka_user)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents = True,
owner='root',
group='root'
)
File(os.path.join(params.limits_conf_dir, 'kafka.conf'),
owner='root',
group='root',
mode=0644,
content=Template("kafka.conf.j2")
)
File(os.path.join(params.conf_dir, 'tools-log4j.properties'),
owner='root',
group='root',
mode=0644,
content=Template("tools-log4j.properties.j2")
)
setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir)
setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)
def _prepare_tez_tarball():
"""
Prepares the Tez tarball by adding the Hadoop native libraries found in the mapreduce tarball.
It's very important to use the version of mapreduce which matches tez here.
Additionally, this will also copy native LZO to the tez tarball if LZO is enabled and the
GPL license has been accepted.
:return: the full path of the newly created tez tarball to use
"""
import tempfile
Logger.info("Preparing the Tez tarball...")
# get the mapreduce tarball which matches the version of tez
# tez installs the mapreduce tar, so it should always be present
_, mapreduce_source_file, _, _ = get_tarball_paths("mapreduce")
_, tez_source_file, _, _ = get_tarball_paths("tez")
temp_dir = Script.get_tmp_dir()
# create the temp staging directories ensuring that non-root agents using tarfile can work with them
mapreduce_temp_dir = tempfile.mkdtemp(prefix="mapreduce-tarball-",
dir=temp_dir)
tez_temp_dir = tempfile.mkdtemp(prefix="tez-tarball-", dir=temp_dir)
sudo.chmod(mapreduce_temp_dir, 0777)
sudo.chmod(tez_temp_dir, 0777)
Logger.info("Extracting {0} to {1}".format(mapreduce_source_file,
mapreduce_temp_dir))
tar_archive.extract_archive(mapreduce_source_file, mapreduce_temp_dir)
Logger.info("Extracting {0} to {1}".format(tez_source_file, tez_temp_dir))
tar_archive.untar_archive(tez_source_file, tez_temp_dir)
hadoop_lib_native_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib",
"native")
tez_lib_dir = os.path.join(tez_temp_dir, "lib")
if not os.path.exists(hadoop_lib_native_dir):
raise Fail(
"Unable to seed the Tez tarball with native libraries since the source Hadoop native lib directory {0} does not exist"
.format(hadoop_lib_native_dir))
if not os.path.exists(tez_lib_dir):
raise Fail(
"Unable to seed the Tez tarball with native libraries since the target Tez lib directory {0} does not exist"
.format(tez_lib_dir))
# copy native libraries from hadoop to tez
Execute(("cp", "-a", hadoop_lib_native_dir, tez_lib_dir), sudo=True)
# if enabled, LZO GPL libraries must be copied as well
if lzo_utils.should_install_lzo():
stack_root = Script.get_stack_root()
service_version = component_version.get_component_repository_version(
service_name="TEZ")
# some installations might not have Tez, but MapReduce2 should be a fallback to get the LZO libraries from
if service_version is None:
Logger.warning(
"Tez does not appear to be installed, using the MapReduce version to get the LZO libraries"
)
service_version = component_version.get_component_repository_version(
service_name="MAPREDUCE2")
hadoop_lib_native_lzo_dir = os.path.join(stack_root, service_version,
"hadoop", "lib", "native")
if not sudo.path_isdir(hadoop_lib_native_lzo_dir):
Logger.warning(
"Unable to located native LZO libraries at {0}, falling back to hadoop home"
.format(hadoop_lib_native_lzo_dir))
hadoop_lib_native_lzo_dir = os.path.join(stack_root, "current",
"hadoop-client", "lib",
"native")
if not sudo.path_isdir(hadoop_lib_native_lzo_dir):
raise Fail(
"Unable to seed the Tez tarball with native libraries since LZO is enabled but the native LZO libraries could not be found at {0}"
.format(hadoop_lib_native_lzo_dir))
Execute(("cp", "-a", hadoop_lib_native_lzo_dir, tez_lib_dir),
sudo=True)
# ensure that the tez/lib directory is readable by non-root (which it typically is not)
Directory(tez_lib_dir, mode=0755, cd_access='a', recursive_ownership=True)
# create the staging directory so that non-root agents can write to it
tez_native_tarball_staging_dir = os.path.join(
temp_dir, "tez-native-tarball-staging")
if not os.path.exists(tez_native_tarball_staging_dir):
Directory(tez_native_tarball_staging_dir,
mode=0777,
cd_access='a',
create_parents=True,
recursive_ownership=True)
tez_tarball_with_native_lib = os.path.join(tez_native_tarball_staging_dir,
"tez-native.tar.gz")
Logger.info("Creating a new Tez tarball at {0}".format(
tez_tarball_with_native_lib))
# tar up Tez, making sure to specify nothing for the arcname so that it does not include an absolute path
with closing(tarfile.open(tez_tarball_with_native_lib,
"w:gz")) as new_tez_tarball:
new_tez_tarball.add(tez_temp_dir, arcname=os.path.sep)
# ensure that the tarball can be read and uploaded
sudo.chmod(tez_tarball_with_native_lib, 0744)
# cleanup
sudo.rmtree(mapreduce_temp_dir)
sudo.rmtree(tez_temp_dir)
return tez_tarball_with_native_lib
def ams(name=None, action=None):
import params
if name == 'collector':
Directory(
params.ams_collector_conf_dir,
owner=params.ams_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True,
)
Directory(params.ams_checkpoint_dir,
owner=params.ams_user,
group=params.user_group,
cd_access="a",
create_parents=True,
recursive_ownership=True)
new_ams_site = {}
new_ams_site.update(params.config['configurations']['ams-site'])
if params.clusterHostInfoDict:
master_components = []
slave_components = []
components = dict(params.clusterHostInfoDict).keys()
known_slave_components = [
"nodemanager", "metrics_monitor", "datanode",
"hbase_regionserver"
]
for component in components:
if component and component.endswith("_hosts"):
component_name = component[:-6]
elif component and component.endswith("_host"):
component_name = component[:-5]
else:
continue
if component_name in known_slave_components:
slave_components.append(component_name)
else:
master_components.append(component_name)
if slave_components:
new_ams_site[
'timeline.metrics.initial.configured.slave.components'] = ",".join(
slave_components)
if master_components:
if 'ambari_server' not in master_components:
master_components.append('ambari_server')
new_ams_site[
'timeline.metrics.initial.configured.master.components'] = ",".join(
master_components)
hbase_total_heapsize_with_trailing_m = params.hbase_heapsize
hbase_total_heapsize = int(
hbase_total_heapsize_with_trailing_m[:-1]) * 1024 * 1024
new_ams_site['hbase_total_heapsize'] = hbase_total_heapsize
XmlConfig(
"ams-site.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=new_ams_site,
configuration_attributes=params.config['configurationAttributes']
['ams-site'],
owner=params.ams_user,
group=params.user_group)
XmlConfig(
"ssl-server.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=params.config['configurations']['ams-ssl-server'],
configuration_attributes=params.config['configurationAttributes']
['ams-ssl-server'],
owner=params.ams_user,
group=params.user_group)
merged_ams_hbase_site = {}
merged_ams_hbase_site.update(
params.config['configurations']['ams-hbase-site'])
if params.security_enabled:
merged_ams_hbase_site.update(
params.config['configurations']['ams-hbase-security-site'])
# Add phoenix client side overrides
merged_ams_hbase_site['phoenix.query.maxGlobalMemoryPercentage'] = str(
params.phoenix_max_global_mem_percent)
merged_ams_hbase_site[
'phoenix.spool.directory'] = params.phoenix_client_spool_dir
XmlConfig(
"hbase-site.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=merged_ams_hbase_site,
configuration_attributes=params.config['configurationAttributes']
['ams-hbase-site'],
owner=params.ams_user,
group=params.user_group)
if params.security_enabled:
TemplateConfig(os.path.join(params.hbase_conf_dir,
"ams_collector_jaas.conf"),
owner=params.ams_user,
template_tag=None)
if (params.log4j_props != None):
File(format("{params.ams_collector_conf_dir}/log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.ams_user,
content=InlineTemplate(params.log4j_props))
File(format("{ams_collector_conf_dir}/ams-env.sh"),
owner=params.ams_user,
content=InlineTemplate(params.ams_env_sh_template))
Directory(
params.ams_collector_log_dir,
owner=params.ams_user,
group=params.user_group,
cd_access="a",
create_parents=True,
mode=0755,
)
Directory(
params.ams_collector_pid_dir,
owner=params.ams_user,
group=params.user_group,
cd_access="a",
create_parents=True,
mode=0755,
)
# Hack to allow native HBase libs to be included for embedded hbase
File(os.path.join(params.ams_hbase_home_dir, "bin", "hadoop"),
owner=params.ams_user,
mode=0755)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
# Setting up security limits
File(os.path.join(params.limits_conf_dir, 'ams.conf'),
owner='root',
group='root',
mode=0644,
content=Template("ams.conf.j2"))
# Phoenix spool file dir if not /tmp
if not os.path.exists(params.phoenix_client_spool_dir):
Directory(params.phoenix_client_spool_dir,
owner=params.ams_user,
mode=0755,
group=params.user_group,
cd_access="a",
create_parents=True)
pass
if not params.is_local_fs_rootdir and params.is_ams_distributed:
# Configuration needed to support NN HA
XmlConfig(
"hdfs-site.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.
config['configurationAttributes']['hdfs-site'],
owner=params.ams_user,
group=params.user_group,
mode=0644)
XmlConfig(
"hdfs-site.xml",
conf_dir=params.hbase_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.
config['configurationAttributes']['hdfs-site'],
owner=params.ams_user,
group=params.user_group,
mode=0644)
# Remove spnego configs from core-site if platform does not have python-kerberos library
truncated_core_site = {}
truncated_core_site.update(
params.config['configurations']['core-site'])
if is_spnego_enabled(params):
truncated_core_site.pop('hadoop.http.authentication.type')
truncated_core_site.pop('hadoop.http.filter.initializers')
XmlConfig("core-site.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=truncated_core_site,
configuration_attributes=params.
config['configurationAttributes']['core-site'],
owner=params.ams_user,
group=params.user_group,
mode=0644)
XmlConfig("core-site.xml",
conf_dir=params.hbase_conf_dir,
configurations=truncated_core_site,
configuration_attributes=params.
config['configurationAttributes']['core-site'],
owner=params.ams_user,
group=params.user_group,
mode=0644)
if params.metric_collector_https_enabled:
export_ca_certs(params.ams_collector_conf_dir)
pass
elif name == 'monitor':
# TODO Uncomment when SPNEGO support has been added to AMS service check and Grafana.
if is_spnego_enabled(params) and is_redhat_centos_6_plus():
try:
import kerberos
except ImportError:
raise ImportError(
"python-kerberos package need to be installed to run AMS in SPNEGO mode"
)
Directory(params.ams_monitor_conf_dir,
owner=params.ams_user,
group=params.user_group,
create_parents=True)
Directory(params.ams_monitor_log_dir,
owner=params.ams_user,
group=params.user_group,
mode=0755,
create_parents=True)
if params.host_in_memory_aggregation and params.log4j_props is not None:
File(format("{params.ams_monitor_conf_dir}/log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.ams_user,
content=InlineTemplate(params.log4j_props))
XmlConfig(
"ams-site.xml",
conf_dir=params.ams_monitor_conf_dir,
configurations=params.config['configurations']['ams-site'],
configuration_attributes=params.
config['configurationAttributes']['ams-site'],
owner=params.ams_user,
group=params.user_group)
XmlConfig("ssl-server.xml",
conf_dir=params.ams_monitor_conf_dir,
configurations=params.config['configurations']
['ams-ssl-server'],
configuration_attributes=params.
config['configurationAttributes']['ams-ssl-server'],
owner=params.ams_user,
group=params.user_group)
pass
Execute(
format(
"{sudo} chown -R {ams_user}:{user_group} {ams_monitor_log_dir}"
))
Directory(params.ams_monitor_pid_dir,
owner=params.ams_user,
group=params.user_group,
cd_access="a",
mode=0755,
create_parents=True)
Directory(format("{ams_monitor_dir}/psutil/build"),
owner=params.ams_user,
group=params.user_group,
cd_access="a",
create_parents=True)
Execute(
format(
"{sudo} chown -R {ams_user}:{user_group} {ams_monitor_dir}"))
TemplateConfig(format("{ams_monitor_conf_dir}/metric_monitor.ini"),
owner=params.ams_user,
group=params.user_group,
template_tag=None)
TemplateConfig(format("{ams_monitor_conf_dir}/metric_groups.conf"),
owner=params.ams_user,
group=params.user_group,
template_tag=None)
File(format("{ams_monitor_conf_dir}/ams-env.sh"),
owner=params.ams_user,
content=InlineTemplate(params.ams_env_sh_template))
if params.metric_collector_https_enabled or params.is_aggregation_https_enabled:
export_ca_certs(params.ams_monitor_conf_dir)
pass
elif name == 'grafana':
ams_grafana_directories = [
params.ams_grafana_conf_dir, params.ams_grafana_log_dir,
params.ams_grafana_data_dir, params.ams_grafana_pid_dir
]
for ams_grafana_directory in ams_grafana_directories:
Directory(ams_grafana_directory,
owner=params.ams_user,
group=params.user_group,
mode=0755,
create_parents=True,
recursive_ownership=True)
File(format("{ams_grafana_conf_dir}/ams-grafana-env.sh"),
owner=params.ams_user,
group=params.user_group,
content=InlineTemplate(params.ams_grafana_env_sh_template))
File(format("{ams_grafana_conf_dir}/ams-grafana.ini"),
owner=params.ams_user,
group=params.user_group,
content=InlineTemplate(params.ams_grafana_ini_template),
mode=0600)
if action != 'stop':
for dir in ams_grafana_directories:
Execute(('chown', '-R', params.ams_user, dir), sudo=True)
if params.metric_collector_https_enabled:
export_ca_certs(params.ams_grafana_conf_dir)
pass
def setup_usersync(upgrade_type=None):
import params
usersync_home = params.usersync_home
ranger_home = params.ranger_home
ranger_ugsync_conf = params.ranger_ugsync_conf
if not is_empty(
params.ranger_usersync_ldap_ldapbindpassword
) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder':
password_validation(params.ranger_usersync_ldap_ldapbindpassword)
Directory(params.ranger_pid_dir,
mode=0750,
owner=params.unix_user,
group=params.unix_group)
Directory(params.usersync_log_dir,
owner=params.unix_user,
group=params.unix_group,
cd_access='a',
create_parents=True,
mode=0755)
File(format('{ranger_ugsync_conf}/ranger-usersync-env-logdir.sh'),
content=format("export logdir={usersync_log_dir}"),
owner=params.unix_user,
group=params.unix_group,
mode=0755)
Directory(format("{ranger_ugsync_conf}/"), owner=params.unix_user)
if upgrade_type is not None:
src_file = format(
'{usersync_home}/conf.dist/ranger-ugsync-default.xml')
dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
if params.stack_supports_ranger_log4j:
File(format('{usersync_home}/conf/log4j.properties'),
owner=params.unix_user,
group=params.unix_group,
content=params.usersync_log4j,
mode=0644)
elif upgrade_type is not None and not params.stack_supports_ranger_log4j:
src_file = format('{usersync_home}/conf.dist/log4j.xml')
dst_file = format('{usersync_home}/conf/log4j.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
XmlConfig(
"ranger-ugsync-site.xml",
conf_dir=ranger_ugsync_conf,
configurations=params.config['configurations']['ranger-ugsync-site'],
configuration_attributes=params.config['configuration_attributes']
['ranger-ugsync-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
if os.path.isfile(params.ranger_ugsync_default_file):
File(params.ranger_ugsync_default_file,
owner=params.unix_user,
group=params.unix_group)
if os.path.isfile(params.usgsync_log4j_file):
File(params.usgsync_log4j_file,
owner=params.unix_user,
group=params.unix_group)
if os.path.isfile(params.cred_validator_file):
File(params.cred_validator_file, group=params.unix_group, mode=04555)
ranger_credential_helper(params.ugsync_cred_lib,
'usersync.ssl.key.password',
params.ranger_usersync_keystore_password,
params.ugsync_jceks_path)
if not is_empty(
params.ranger_usersync_ldap_ldapbindpassword
) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder':
ranger_credential_helper(params.ugsync_cred_lib,
'ranger.usersync.ldap.bindalias',
params.ranger_usersync_ldap_ldapbindpassword,
params.ugsync_jceks_path)
ranger_credential_helper(params.ugsync_cred_lib,
'usersync.ssl.truststore.password',
params.ranger_usersync_truststore_password,
params.ugsync_jceks_path)
File(params.ugsync_jceks_path,
owner=params.unix_user,
group=params.unix_group,
mode=0640)
File([params.usersync_start, params.usersync_stop],
owner=params.unix_user,
group=params.unix_group)
File(
params.usersync_services_file,
mode=0755,
)
Execute(('ln', '-sf', format('{usersync_services_file}'),
'/usr/bin/ranger-usersync'),
not_if=format("ls /usr/bin/ranger-usersync"),
only_if=format("ls {usersync_services_file}"),
sudo=True)
if not os.path.isfile(params.ranger_usersync_keystore_file):
cmd = format(
"{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'"
)
Execute(cmd, logoutput=True, user=params.unix_user)
File(params.ranger_usersync_keystore_file,
owner=params.unix_user,
group=params.unix_group,
mode=0640)
create_core_site_xml(ranger_ugsync_conf)
def ams(name=None):
import params
if name == 'collector':
if not check_windows_service_exists(
params.ams_collector_win_service_name):
Execute(
format(
"cmd /C cd {ams_collector_home_dir} & ambari-metrics-collector.cmd setup"
))
Directory(params.ams_collector_conf_dir,
owner=params.ams_user,
create_parents=True)
Directory(params.ams_checkpoint_dir,
owner=params.ams_user,
create_parents=True)
XmlConfig(
"ams-site.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=params.config['configurations']['ams-site'],
configuration_attributes=params.config['configurationAttributes']
['ams-site'],
owner=params.ams_user,
)
merged_ams_hbase_site = {}
merged_ams_hbase_site.update(
params.config['configurations']['ams-hbase-site'])
if params.security_enabled:
merged_ams_hbase_site.update(
params.config['configurations']['ams-hbase-security-site'])
XmlConfig(
"hbase-site.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=merged_ams_hbase_site,
configuration_attributes=params.config['configurationAttributes']
['ams-hbase-site'],
owner=params.ams_user,
)
if (params.log4j_props != None):
File(os.path.join(params.ams_collector_conf_dir,
"log4j.properties"),
owner=params.ams_user,
content=params.log4j_props)
File(os.path.join(params.ams_collector_conf_dir, "ams-env.cmd"),
owner=params.ams_user,
content=InlineTemplate(params.ams_env_sh_template))
ServiceConfig(params.ams_collector_win_service_name,
action="change_user",
username=params.ams_user,
password=Script.get_password(params.ams_user))
if not params.is_local_fs_rootdir:
# Configuration needed to support NN HA
XmlConfig(
"hdfs-site.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.
config['configurationAttributes']['hdfs-site'],
owner=params.ams_user,
group=params.user_group,
mode=0644)
XmlConfig(
"hdfs-site.xml",
conf_dir=params.hbase_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.
config['configurationAttributes']['hdfs-site'],
owner=params.ams_user,
group=params.user_group,
mode=0644)
XmlConfig(
"core-site.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.
config['configurationAttributes']['core-site'],
owner=params.ams_user,
group=params.user_group,
mode=0644)
XmlConfig(
"core-site.xml",
conf_dir=params.hbase_conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.
config['configurationAttributes']['core-site'],
owner=params.ams_user,
group=params.user_group,
mode=0644)
else:
ServiceConfig(params.ams_embedded_hbase_win_service_name,
action="change_user",
username=params.ams_user,
password=Script.get_password(params.ams_user))
# creating symbolic links on ams jars to make them available to services
links_pairs = [
("%COLLECTOR_HOME%\\hbase\\lib\\ambari-metrics-hadoop-sink-with-common.jar",
"%SINK_HOME%\\hadoop-sink\\ambari-metrics-hadoop-sink-with-common-*.jar"
),
]
for link_pair in links_pairs:
link, target = link_pair
real_link = os.path.expandvars(link)
target = compress_backslashes(
glob.glob(os.path.expandvars(target))[0])
if not os.path.exists(real_link):
#TODO check the symlink destination too. Broken in Python 2.x on Windows.
Execute('cmd /c mklink "{0}" "{1}"'.format(
real_link, target))
pass
elif name == 'monitor':
if not check_windows_service_exists(
params.ams_monitor_win_service_name):
Execute(
format(
"cmd /C cd {ams_monitor_home_dir} & ambari-metrics-monitor.cmd setup"
))
# creating symbolic links on ams jars to make them available to services
links_pairs = [
("%HADOOP_HOME%\\share\\hadoop\\common\\lib\\ambari-metrics-hadoop-sink-with-common.jar",
"%SINK_HOME%\\hadoop-sink\\ambari-metrics-hadoop-sink-with-common-*.jar"
),
("%HBASE_HOME%\\lib\\ambari-metrics-hadoop-sink-with-common.jar",
"%SINK_HOME%\\hadoop-sink\\ambari-metrics-hadoop-sink-with-common-*.jar"
),
]
for link_pair in links_pairs:
link, target = link_pair
real_link = os.path.expandvars(link)
target = compress_backslashes(
glob.glob(os.path.expandvars(target))[0])
if not os.path.exists(real_link):
#TODO check the symlink destination too. Broken in Python 2.x on Windows.
Execute('cmd /c mklink "{0}" "{1}"'.format(real_link, target))
Directory(params.ams_monitor_conf_dir,
owner=params.ams_user,
create_parents=True)
if params.host_in_memory_aggregation:
if params.log4j_props is not None:
File(os.path.join(params.ams_monitor_conf_dir,
"log4j.properties"),
owner=params.ams_user,
content=params.log4j_props)
pass
XmlConfig(
"ams-site.xml",
conf_dir=params.ams_monitor_conf_dir,
configurations=params.config['configurations']['ams-site'],
configuration_attributes=params.
config['configurationAttributes']['ams-site'],
owner=params.ams_user,
group=params.user_group)
XmlConfig("ssl-server.xml",
conf_dir=params.ams_monitor_conf_dir,
configurations=params.config['configurations']
['ams-ssl-server'],
configuration_attributes=params.
config['configurationAttributes']['ams-ssl-server'],
owner=params.ams_user,
group=params.user_group)
pass
TemplateConfig(os.path.join(params.ams_monitor_conf_dir,
"metric_monitor.ini"),
owner=params.ams_user,
template_tag=None)
TemplateConfig(os.path.join(params.ams_monitor_conf_dir,
"metric_groups.conf"),
owner=params.ams_user,
template_tag=None)
ServiceConfig(params.ams_monitor_win_service_name,
action="change_user",
username=params.ams_user,
password=Script.get_password(params.ams_user))
def setup_ranger_admin(upgrade_type=None):
import params
ranger_home = params.ranger_home
ranger_conf = params.ranger_conf
Directory(ranger_conf,
owner=params.unix_user,
group=params.unix_group,
create_parents=True)
if upgrade_type is not None:
copy_jdbc_connector()
File(
format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}{check_db_connection_jar_name}")),
mode=0644,
)
cp = format("{check_db_connection_jar}")
if params.db_flavor.lower() == 'sqla':
cp = cp + os.pathsep + format("{ranger_home}/ews/lib/sajdbc4.jar")
else:
cp = cp + os.pathsep + format("{driver_curl_target}")
cp = cp + os.pathsep + format("{ranger_home}/ews/lib/*")
db_connection_check_command = format(
"{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_jdbc_connection_url}' {ranger_db_user} {ranger_db_password!p} {ranger_jdbc_driver}"
)
env_dict = {}
if params.db_flavor.lower() == 'sqla':
env_dict = {'LD_LIBRARY_PATH': params.ld_lib_path}
Execute(db_connection_check_command,
path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
tries=5,
try_sleep=10,
environment=env_dict)
Execute(
('ln', '-sf', format('{ranger_home}/ews/webapp/WEB-INF/classes/conf'),
format('{ranger_home}/conf')),
not_if=format("ls {ranger_home}/conf"),
only_if=format("ls {ranger_home}/ews/webapp/WEB-INF/classes/conf"),
sudo=True)
if upgrade_type is not None:
src_file = format(
'{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/ranger-admin-default-site.xml'
)
dst_file = format('{ranger_home}/conf/ranger-admin-default-site.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
src_file = format(
'{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/security-applicationContext.xml'
)
dst_file = format('{ranger_home}/conf/security-applicationContext.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
Directory(
format('{ranger_home}/'),
owner=params.unix_user,
group=params.unix_group,
recursive_ownership=True,
)
Directory(params.admin_log_dir,
owner=params.unix_user,
group=params.unix_group,
create_parents=True,
cd_access='a',
mode=0755)
File(format('{ranger_conf}/ranger-admin-env-logdir.sh'),
content=format("export RANGER_ADMIN_LOG_DIR={admin_log_dir}"),
owner=params.unix_user,
group=params.unix_group,
mode=0755)
if os.path.isfile(params.ranger_admin_default_file):
File(params.ranger_admin_default_file,
owner=params.unix_user,
group=params.unix_group)
else:
Logger.warning(
'Required file {0} does not exist, copying the file to {1} path'.
format(params.ranger_admin_default_file, ranger_conf))
src_file = format(
'{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/ranger-admin-default-site.xml'
)
dst_file = format('{ranger_home}/conf/ranger-admin-default-site.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
File(params.ranger_admin_default_file,
owner=params.unix_user,
group=params.unix_group)
if os.path.isfile(params.security_app_context_file):
File(params.security_app_context_file,
owner=params.unix_user,
group=params.unix_group)
else:
Logger.warning(
'Required file {0} does not exist, copying the file to {1} path'.
format(params.security_app_context_file, ranger_conf))
src_file = format(
'{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/security-applicationContext.xml'
)
dst_file = format('{ranger_home}/conf/security-applicationContext.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
File(params.security_app_context_file,
owner=params.unix_user,
group=params.unix_group)
if upgrade_type is not None and params.stack_supports_config_versioning:
if os.path.islink('/usr/bin/ranger-admin'):
Link('/usr/bin/ranger-admin', action="delete")
Link('/usr/bin/ranger-admin',
to=format('{ranger_home}/ews/ranger-admin-services.sh'))
Execute(('ln', '-sf', format('{ranger_home}/ews/ranger-admin-services.sh'),
'/usr/bin/ranger-admin'),
not_if=format("ls /usr/bin/ranger-admin"),
only_if=format("ls {ranger_home}/ews/ranger-admin-services.sh"),
sudo=True)
XmlConfig(
"ranger-admin-site.xml",
conf_dir=ranger_conf,
configurations=params.config['configurations']['ranger-admin-site'],
configuration_attributes=params.config['configuration_attributes']
['ranger-admin-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
Directory(
os.path.join(ranger_conf, 'ranger_jaas'),
mode=0700,
owner=params.unix_user,
group=params.unix_group,
)
if params.stack_supports_ranger_log4j:
File(format('{ranger_home}/ews/webapp/WEB-INF/log4j.properties'),
owner=params.unix_user,
group=params.unix_group,
content=params.admin_log4j,
mode=0644)
do_keystore_setup(upgrade_type=upgrade_type)
create_core_site_xml(ranger_conf)
def upgrade_schema(self, env):
"""
Executes the schema upgrade binary. This is its own function because it could
be called as a standalone task from the upgrade pack, but is safe to run it for each
metastore instance. The schema upgrade on an already upgraded metastore is a NOOP.
The metastore schema upgrade requires a database driver library for most
databases. During an upgrade, it's possible that the library is not present,
so this will also attempt to copy/download the appropriate driver.
This function will also ensure that configurations are written out to disk before running
since the new configs will most likely not yet exist on an upgrade.
Should not be invoked for a DOWNGRADE; Metastore only supports schema upgrades.
"""
Logger.info("Upgrading Hive Metastore Schema")
import status_params
import params
env.set_params(params)
# ensure that configurations are written out before trying to upgrade the schema
# since the schematool needs configs and doesn't know how to use the hive conf override
self.configure(env)
if params.security_enabled:
cached_kinit_executor(status_params.kinit_path_local,
status_params.hive_user,
params.hive_metastore_keytab_path,
params.hive_metastore_principal,
status_params.hostname,
status_params.tmp_dir)
# ensure that the JDBC drive is present for the schema tool; if it's not
# present, then download it first
if params.hive_jdbc_driver in params.hive_jdbc_drivers_list:
target_directory = format("{stack_root}/{version}/hive/lib")
# download it if it does not exist
if not os.path.exists(params.source_jdbc_file):
jdbc_connector(params.hive_jdbc_target,
params.hive_previous_jdbc_jar)
target_directory_and_filename = os.path.join(
target_directory, os.path.basename(params.source_jdbc_file))
if params.sqla_db_used:
target_native_libs_directory = format(
"{target_directory}/native/lib64")
Execute(
format(
"yes | {sudo} cp {jars_in_hive_lib} {target_directory}"
))
Directory(target_native_libs_directory, create_parents=True)
Execute(
format(
"yes | {sudo} cp {libs_in_hive_lib} {target_native_libs_directory}"
))
Execute(
format(
"{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*"
))
else:
# copy the JDBC driver from the older metastore location to the new location only
# if it does not already exist
if not os.path.exists(target_directory_and_filename):
Execute(('cp', params.source_jdbc_file, target_directory),
path=["/bin", "/usr/bin/"],
sudo=True)
File(target_directory_and_filename, mode=0644)
# build the schema tool command
binary = format("{hive_schematool_ver_bin}/schematool")
# the conf.server directory changed locations between stack versions
# since the configurations have not been written out yet during an upgrade
# we need to choose the original legacy location
schematool_hive_server_conf_dir = params.hive_server_conf_dir
upgrade_from_version = upgrade_summary.get_source_version(
"HIVE", default_version=params.version_for_stack_feature_checks)
if not (check_stack_feature(StackFeature.CONFIG_VERSIONING,
upgrade_from_version)):
schematool_hive_server_conf_dir = LEGACY_HIVE_SERVER_CONF
env_dict = {'HIVE_CONF_DIR': schematool_hive_server_conf_dir}
command = format(
"{binary} -dbType {hive_metastore_db_type} -upgradeSchema")
Execute(command,
user=params.hive_user,
tries=1,
environment=env_dict,
logoutput=True)
def create_dir_as_hawq_user(directory): """ Creates directories with hawq_user and hawq_group (defaults to gpadmin:gpadmin) """ Directory(directory, create_parents = True, owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group)
def setup_logfeeder():
import params
Directory([params.logfeeder_log_dir, params.logfeeder_pid_dir, params.logfeeder_checkpoint_folder],
mode=0755,
cd_access='a',
create_parents=True
)
Directory([params.logfeeder_dir, params.logsearch_logfeeder_conf],
mode=0755,
cd_access='a',
create_parents=True,
recursive_ownership=True
)
File(params.logfeeder_log,
mode=0644,
content=''
)
PropertiesFile(format("{logsearch_logfeeder_conf}/logfeeder.properties"),
properties = params.logfeeder_properties
)
File(format("{logsearch_logfeeder_conf}/logfeeder-env.sh"),
content=InlineTemplate(params.logfeeder_env_content),
mode=0755
)
File(format("{logsearch_logfeeder_conf}/log4j.xml"),
content=InlineTemplate(params.logfeeder_log4j_content)
)
File(format("{logsearch_logfeeder_conf}/grok-patterns"),
content=InlineTemplate(params.logfeeder_grok_patterns),
encoding="utf-8"
)
for file_name in params.logfeeder_default_config_file_names:
File(format("{logsearch_logfeeder_conf}/" + file_name),
content=Template(file_name + ".j2")
)
for service, pattern_content in params.logfeeder_metadata.iteritems():
File(format("{logsearch_logfeeder_conf}/input.config-" + service.replace('-logsearch-conf', '') + ".json"),
content=InlineTemplate(pattern_content, extra_imports=[default])
)
if params.logfeeder_system_log_enabled:
File(format("{logsearch_logfeeder_conf}/input.config-system_messages.json"),
content=params.logfeeder_system_messages_content
)
File(format("{logsearch_logfeeder_conf}/input.config-secure_log.json"),
content=params.logfeeder_secure_log_content
)
if params.security_enabled:
File(format("{logfeeder_jaas_file}"),
content=Template("logfeeder_jaas.conf.j2")
)
def jdbc_connector(target, hive_previous_jdbc_jar):
"""
Shared by Hive Batch, Hive Metastore, and Hive Interactive
:param target: Target of jdbc jar name, which could be for any of the components above.
"""
import params
if not params.jdbc_jar_name:
return
if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
environment = {"no_proxy": format("{ambari_server_hostname}")}
if hive_previous_jdbc_jar and os.path.isfile(hive_previous_jdbc_jar):
File(hive_previous_jdbc_jar, action='delete')
# TODO: should be removed after ranger_hive_plugin will not provide jdbc
if params.prepackaged_jdbc_name != params.jdbc_jar_name:
Execute(('rm', '-f', params.prepackaged_ojdbc_symlink),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source))
# maybe it will be more correcvly to use db type
if params.sqla_db_used:
untar_sqla_type2_driver = ('tar', '-xvf',
params.downloaded_custom_connector,
'-C', params.tmp_dir)
Execute(untar_sqla_type2_driver, sudo=True)
Execute(
format("yes | {sudo} cp {jars_path_in_archive} {hive_lib}"))
Directory(params.jdbc_libs_dir, create_parents=True)
Execute(
format(
"yes | {sudo} cp {libs_path_in_archive} {jdbc_libs_dir}"))
Execute(
format(
"{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*"))
else:
Execute(
('cp', '--remove-destination',
params.downloaded_custom_connector, target),
#creates=target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
path=["/bin", "/usr/bin/"],
sudo=True)
else:
#for default hive db (Mysql)
Execute(
('cp', '--remove-destination',
format('/usr/share/java/{jdbc_jar_name}'), target),
#creates=target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
path=["/bin", "/usr/bin/"],
sudo=True)
pass
File(
target,
mode=0644,
)
def oozie(is_server=False):
import params
if is_server:
params.HdfsResource(params.oozie_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.oozie_user,
mode=params.oozie_hdfs_user_mode)
params.HdfsResource(None, action="execute")
Directory(params.conf_dir,
recursive=True,
owner=params.oozie_user,
group=params.user_group)
XmlConfig(
"oozie-site.xml",
conf_dir=params.conf_dir,
configurations=params.oozie_site,
configuration_attributes=params.config['configuration_attributes']
['oozie-site'],
owner=params.oozie_user,
group=params.user_group,
mode=0664)
File(
format("{conf_dir}/oozie-env.sh"),
owner=params.oozie_user,
content=InlineTemplate(params.oozie_env_sh_template),
group=params.user_group,
)
if (params.log4j_props != None):
File(format("{params.conf_dir}/oozie-log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.oozie_user,
content=params.log4j_props)
elif (os.path.exists(format("{params.conf_dir}/oozie-log4j.properties"))):
File(format("{params.conf_dir}/oozie-log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.oozie_user)
if params.hdp_stack_version != "" and compare_versions(
params.hdp_stack_version, '2.2') >= 0:
File(format("{params.conf_dir}/adminusers.txt"),
mode=0644,
group=params.user_group,
owner=params.oozie_user,
content=Template('adminusers.txt.j2',
oozie_admin_users=params.oozie_admin_users))
else:
File(format("{params.conf_dir}/adminusers.txt"),
owner=params.oozie_user,
group=params.user_group)
if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \
params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
params.jdbc_driver_name == "org.postgresql.Driver" or \
params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver":
File(
format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}{check_db_connection_jar_name}")),
)
pass
oozie_ownership()
if is_server:
oozie_server_specific()