def configure(self, env, upgrade_type=None):
import params
# The configure command doesn't actually receive the upgrade_type from Script.py, so get it from the config dictionary
if upgrade_type is None:
restart_type = default("/commandParams/restart_type", "")
if restart_type.lower() == "rolling_upgrade":
upgrade_type = UPGRADE_TYPE_ROLLING
elif restart_type.lower() == "nonrolling_upgrade":
upgrade_type = UPGRADE_TYPE_NON_ROLLING
if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and params.version is not None:
Logger.info(format("Configuring Oozie during upgrade type: {upgrade_type}, direction: {params.upgrade_direction}, and version {params.version}"))
if compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
# In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of
# oozie, we need to create the symlinks both for server and client.
# This is required as both need to be pointing to new installed oozie version.
# Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie
stack_select.select("oozie-client", params.version)
# Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie
stack_select.select("oozie-server", params.version)
if compare_versions(format_stack_version(params.version), '2.3.0.0') >= 0:
conf_select.select(params.stack_name, "oozie", params.version)
env.set_params(params)
oozie(is_server=True)
def pre_upgrade_restart(self, env, upgrade_type=None):
"""
Performs the tasks that should be done before an upgrade of oozie. This includes:
- backing up configurations
- running hdp-select and conf-select
- restoring configurations
- preparing the libext directory
:param env:
:return:
"""
import params
env.set_params(params)
# this function should not execute if the version can't be determined or
# is not at least HDP 2.2.0.0
if not params.version or compare_versions(format_stack_version(params.version), '2.2.0.0') < 0:
return
Logger.info("Executing Oozie Server Stack Upgrade pre-restart")
OozieUpgrade.backup_configuration()
if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
conf_select.select(params.stack_name, "oozie", params.version)
stack_select.select("oozie-server", params.version)
OozieUpgrade.restore_configuration()
OozieUpgrade.prepare_libext_directory()
def pre_upgrade_restart(self, env, upgrade_type=None):
"""
Performs the tasks surrounding the Oozie startup when a rolling upgrade
is in progress. This includes backing up the configuration, updating
the database, preparing the WAR, and installing the sharelib in HDFS.
:param env:
:return:
"""
import params
env.set_params(params)
# this function should not execute if the version can't be determined or
# is not at least IOP 4.0.0.0
if not params.version or compare_versions(
format_stack_version(params.version), '4.0.0.0') < 0:
return
Logger.info("Executing Oozie Server Rolling Upgrade pre-restart")
oozie_server_upgrade.backup_configuration()
stack_select.select_packages(params.version)
#Execute(format("iop-select set oozie-server {version}"))
oozie_server_upgrade.restore_configuration()
#oozie_server_upgrade.prepare_libext_directory()
oozie_server_upgrade.upgrade_oozie()
def pre_rolling_restart(self, env):
"""
Performs the tasks surrounding the Oozie startup when a rolling upgrade
is in progress. This includes backing up the configuration, updating
the database, preparing the WAR, and installing the sharelib in HDFS.
:param env:
:return:
"""
import params
env.set_params(params)
# this function should not execute if the version can't be determined or
# is not at least HDP 2.2.0.0
if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0:
return
Logger.info("Executing Oozie Server Rolling Upgrade pre-restart")
oozie_server_upgrade.backup_configuration()
conf_select.select(params.stack_name, "oozie", params.version)
hdp_select.select("oozie-server", params.version)
oozie_server_upgrade.restore_configuration()
oozie_server_upgrade.prepare_libext_directory()
oozie_server_upgrade.upgrade_oozie()
def prepare_libext_directory():
"""
Creates /usr/iop/current/oozie/libext-customer and recursively sets
777 permissions on it and its parents.
:return:
"""
import params
# some versions of IOP don't need the lzo compression libraries
target_version_needs_compression_libraries = compare_versions(
format_stack_version(params.version), '4.0.0.0') >= 0
if not os.path.isdir(params.oozie_libext_customer_dir):
os.makedirs(params.oozie_libext_customer_dir, 0o777)
# ensure that it's rwx for all
os.chmod(params.oozie_libext_customer_dir, 0o777)
# get all hadooplzo* JAR files
# stack-select set hadoop-client has not run yet, therefore we cannot use
# /usr/iop/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 4.0.0.0; don't do this
# for downgrade to 4.0.0.0 since hadoop-lzo will not be present
# This can also be called during a Downgrade.
# When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE
or target_version_needs_compression_libraries):
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format("/usr/iop/{version}/hadoop/lib")
files = glob.iglob(
os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
if not files:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy files into libext
files_copied = False
for file in files:
if os.path.isfile(file):
Logger.info("Copying {0} to {1}".format(
str(file), params.oozie_libext_customer_dir))
shutil.copy2(file, params.oozie_libext_customer_dir)
files_copied = True
if not files_copied:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy ext ZIP to customer dir
oozie_ext_zip_file = '/usr/share/IOP-oozie/ext-2.2.zip'
if not os.path.isfile(oozie_ext_zip_file):
raise Fail("Unable to copy {0} because it does not exist".format(
oozie_ext_zip_file))
Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file,
params.oozie_libext_customer_dir))
shutil.copy2(oozie_ext_zip_file, params.oozie_libext_customer_dir)
def configure(self, env, upgrade_type=None):
import params
#TODO: needed?
if upgrade_type == "nonrolling" and params.upgrade_direction == Direction.UPGRADE and \
params.version and compare_versions(format_stack_version(params.version), '4.1.0.0') >= 0:
# In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of
# oozie, we need to create the symlinks both for server and client.
# This is required as both need to be pointing to new installed oozie version.
# Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie
# Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie
stack_select.select_packages(params.version)
env.set_params(params)
oozie(is_server=True)
def prepare_libext_directory():
"""
Performs the following actions on libext:
- creates /usr/hdp/current/oozie/libext and recursively
- set 777 permissions on it and its parents.
- downloads JDBC driver JAR if needed
- copies Falcon JAR for the Oozie WAR if needed
"""
import params
# some versions of HDP don't need the lzo compression libraries
target_version_needs_compression_libraries = compare_versions(
format_hdp_stack_version(params.version), '2.2.1.0') >= 0
# ensure the directory exists
Directory(params.oozie_libext_dir, mode = 0777)
# get all hadooplzo* JAR files
# hdp-select set hadoop-client has not run yet, therefore we cannot use
# /usr/hdp/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 2.2.0.0; don't do this
# for downgrade to 2.2.0.0 since hadoop-lzo will not be present
# This can also be called during a Downgrade.
# When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries):
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")
files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
if not files:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy files into libext
files_copied = False
for file in files:
if os.path.isfile(file):
Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_dir))
shutil.copy2(file, params.oozie_libext_dir)
files_copied = True
if not files_copied:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy ext ZIP to libext dir
oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
# something like /usr/hdp/current/oozie-server/libext/ext-2.2.zip
oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, "ext-2.2.zip")
if not os.path.isfile(oozie_ext_zip_file):
raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file))
Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_dir))
Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True)
Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True)
File(oozie_ext_zip_target_path,
mode=0644
)
# Redownload jdbc driver to a new current location
oozie.download_database_library_if_needed()
# get the upgrade version in the event that it's needed
upgrade_stack = hdp_select._get_upgrade_stack()
if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None:
raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.")
# something like 2.3.0.0-1234
stack_version = upgrade_stack[1]
# copy the Falcon JAR if needed; falcon has not upgraded yet, so we must
# use the versioned falcon directory
if params.has_falcon_host:
versioned_falcon_jar_directory = "/usr/hdp/{0}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(stack_version)
Logger.info("Copying {0} to {1}".format(versioned_falcon_jar_directory, params.oozie_libext_dir))
Execute(format('{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}'))
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'))
def storm():
import params
Directory(params.log_dir,
owner=params.storm_user,
group=params.user_group,
mode=0775,
recursive=True)
Directory(
[params.pid_dir, params.local_dir, params.conf_dir],
owner=params.storm_user,
group=params.user_group,
recursive=True,
cd_access="a",
)
File(format("{conf_dir}/config.yaml"),
content=Template("config.yaml.j2"),
owner=params.storm_user,
group=params.user_group)
configurations = params.config['configurations']['storm-site']
File(format("{conf_dir}/storm.yaml"),
content=Template("storm.yaml.j2",
extra_imports=[escape_yaml_propetry],
configurations=configurations),
owner=params.storm_user,
group=params.user_group)
if params.has_metric_collector:
File(format("{conf_dir}/storm-metrics2.properties"),
owner=params.storm_user,
group=params.user_group,
content=Template("storm-metrics2.properties.j2"))
Execute(format(
"sudo ln -s {metric_collector_sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar"
),
not_if=format(
"ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"),
only_if=format("ls {metric_collector_sink_jar}"))
File(format("{conf_dir}/storm-env.sh"),
owner=params.storm_user,
content=InlineTemplate(params.storm_env_sh_template))
if params.security_enabled:
TemplateConfig(format("{conf_dir}/storm_jaas.conf"),
owner=params.storm_user)
if params.hdp_stack_version != "" and compare_versions(
params.hdp_stack_version, '2.2') >= 0:
TemplateConfig(format("{conf_dir}/client_jaas.conf"),
owner=params.storm_user)
minRuid = configurations[
'_storm.min.ruid'] if configurations.has_key(
'_storm.min.ruid') else ''
min_user_ruid = int(
minRuid) if minRuid.isdigit() else _find_real_user_min_uid()
File(format("{conf_dir}/worker-launcher.cfg"),
content=Template("worker-launcher.cfg.j2",
min_user_ruid=min_user_ruid),
owner='root',
group=params.user_group)
def storm(name=None):
import params
Directory(params.log_dir,
owner=params.storm_user,
group=params.user_group,
mode=0777,
recursive=True
)
Directory([params.pid_dir, params.local_dir],
owner=params.storm_user,
group=params.user_group,
recursive=True,
cd_access="a",
mode=0755,
)
Directory(params.conf_dir,
group=params.user_group,
recursive=True,
cd_access="a",
)
File(format("{conf_dir}/config.yaml"),
content=Template("config.yaml.j2"),
owner=params.storm_user,
group=params.user_group
)
configurations = params.config['configurations']['storm-site']
File(format("{conf_dir}/storm.yaml"),
content=yaml_config_template(configurations),
owner=params.storm_user,
group=params.user_group
)
if params.has_metric_collector:
File(format("{conf_dir}/storm-metrics2.properties"),
owner=params.storm_user,
group=params.user_group,
content=Template("storm-metrics2.properties.j2")
)
# Remove symlink. It can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2
Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar",
action="delete")
Execute(format("{sudo} ln -s {metric_collector_sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar"),
not_if=format("ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"),
only_if=format("ls {metric_collector_sink_jar}")
)
File(format("{conf_dir}/storm-env.sh"),
owner=params.storm_user,
content=InlineTemplate(params.storm_env_sh_template)
)
if params.storm_logs_supported:
Directory(params.log4j_dir,
owner=params.storm_user,
group=params.user_group,
mode=0755,
recursive=True
)
File(format("{log4j_dir}/cluster.xml"),
owner=params.storm_user,
content=InlineTemplate(params.storm_cluster_log4j_content)
)
File(format("{log4j_dir}/worker.xml"),
owner=params.storm_user,
content=InlineTemplate(params.storm_worker_log4j_content)
)
if params.security_enabled:
TemplateConfig(format("{conf_dir}/storm_jaas.conf"),
owner=params.storm_user
)
if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0:
TemplateConfig(format("{conf_dir}/client_jaas.conf"),
owner=params.storm_user
)
minRuid = configurations['_storm.min.ruid'] if configurations.has_key('_storm.min.ruid') else ''
min_user_ruid = int(minRuid) if minRuid.isdigit() else _find_real_user_min_uid()
File(format("{conf_dir}/worker-launcher.cfg"),
content=Template("worker-launcher.cfg.j2", min_user_ruid = min_user_ruid),
owner='root',
group=params.user_group
)
def oozie(is_server=False):
import params
if is_server:
params.HdfsDirectory(params.oozie_hdfs_user_dir,
action="create",
owner=params.oozie_user,
mode=params.oozie_hdfs_user_mode)
Directory(params.conf_dir,
recursive=True,
owner=params.oozie_user,
group=params.user_group)
XmlConfig(
"oozie-site.xml",
conf_dir=params.conf_dir,
configurations=params.oozie_site,
configuration_attributes=params.config['configuration_attributes']
['oozie-site'],
owner=params.oozie_user,
group=params.user_group,
mode=0664)
File(format("{conf_dir}/oozie-env.sh"),
owner=params.oozie_user,
content=InlineTemplate(params.oozie_env_sh_template))
if (params.log4j_props != None):
File(format("{params.conf_dir}/oozie-log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.oozie_user,
content=params.log4j_props)
elif (os.path.exists(format("{params.conf_dir}/oozie-log4j.properties"))):
File(format("{params.conf_dir}/oozie-log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.oozie_user)
if params.hdp_stack_version != "" and compare_versions(
params.hdp_stack_version, '2.2') >= 0:
File(format("{params.conf_dir}/adminusers.txt"),
mode=0644,
group=params.user_group,
owner=params.oozie_user,
content=Template('adminusers.txt.j2',
oozie_user=params.oozie_user))
else:
File(format("{params.conf_dir}/adminusers.txt"),
owner=params.oozie_user,
group=params.user_group)
if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \
params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
params.jdbc_driver_name == "org.postgresql.Driver" or \
params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver":
File(
format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}{check_db_connection_jar_name}")),
)
pass
oozie_ownership()
if is_server:
oozie_server_specific()
def oozie_server_specific():
import params
File(
params.pid_file,
action="delete",
not_if=
"ls {pid_file} >/dev/null 2>&1 && !(ps `cat {pid_file}` >/dev/null 2>&1)"
)
oozie_server_directories = [
format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir,
params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir,
params.oozie_lib_dir, params.oozie_webapps_dir,
params.oozie_webapps_conf_dir, params.oozie_server_dir
]
Directory(
oozie_server_directories,
owner=params.oozie_user,
group=params.user_group,
mode=0755,
recursive=True,
cd_access="a",
)
Directory(
params.oozie_libext_dir,
recursive=True,
)
configure_cmds = []
configure_cmds.append(
('tar', '-xvf', format('{oozie_home}/oozie-sharelib.tar.gz'), '-C',
params.oozie_home))
configure_cmds.append(('cp', params.ext_js_path, params.oozie_libext_dir))
configure_cmds.append(('chown', format('{oozie_user}:{user_group}'),
format('{oozie_libext_dir}/{ext_js_file}')))
configure_cmds.append(('chown', '-RL', format('{oozie_user}:{user_group}'),
params.oozie_webapps_conf_dir))
no_op_test = format(
"ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"
)
Execute(
configure_cmds,
not_if=no_op_test,
sudo=True,
)
if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \
params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver":
File(
params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source),
)
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
params.target),
#creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.target, owner=params.oozie_user, group=params.user_group)
#falcon el extension
if params.has_falcon_host:
Execute(
format(
'{sudo} cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}'
),
not_if=no_op_test,
)
Execute(
format(
'{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'
),
not_if=no_op_test,
)
if params.lzo_enabled and len(params.all_lzo_packages) > 0:
Package(params.all_lzo_packages)
Execute(
format(
'{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
not_if=no_op_test,
)
Execute(format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war"),
user=params.oozie_user,
not_if=no_op_test)
if params.hdp_stack_version != "" and compare_versions(
params.hdp_stack_version, '2.2') >= 0:
# Create hive-site and tez-site configs for oozie
Directory(params.hive_conf_dir,
recursive=True,
owner=params.oozie_user,
group=params.user_group)
if 'hive-site' in params.config['configurations']:
XmlConfig(
"hive-site.xml",
conf_dir=params.hive_conf_dir,
configurations=params.config['configurations']['hive-site'],
configuration_attributes=params.
config['configuration_attributes']['hive-site'],
owner=params.oozie_user,
group=params.user_group,
mode=0644)
if 'tez-site' in params.config['configurations']:
XmlConfig(
"tez-site.xml",
conf_dir=params.hive_conf_dir,
configurations=params.config['configurations']['tez-site'],
configuration_attributes=params.
config['configuration_attributes']['tez-site'],
owner=params.oozie_user,
group=params.user_group,
mode=0664)
Execute(('chown', '-R', format("{oozie_user}:{user_group}"),
params.oozie_server_dir),
sudo=True)
def prepare_libext_directory():
"""
Creates /usr/hdp/current/oozie/libext-customer and recursively sets
777 permissions on it and its parents.
Also, downloads jdbc driver and provides other staff
"""
import params
# some versions of HDP don't need the lzo compression libraries
target_version_needs_compression_libraries = compare_versions(
format_hdp_stack_version(params.version), '2.2.1.0') >= 0
if not os.path.isdir(params.oozie_libext_customer_dir):
os.makedirs(params.oozie_libext_customer_dir, 0o777)
# ensure that it's rwx for all
os.chmod(params.oozie_libext_customer_dir, 0o777)
# get all hadooplzo* JAR files
# hdp-select set hadoop-client has not run yet, therefore we cannot use
# /usr/hdp/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 2.2.0.0; don't do this
# for downgrade to 2.2.0.0 since hadoop-lzo will not be present
# This can also be called during a Downgrade.
# When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE
or target_version_needs_compression_libraries):
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")
files = glob.iglob(
os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
if not files:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy files into libext
files_copied = False
for file in files:
if os.path.isfile(file):
Logger.info("Copying {0} to {1}".format(
str(file), params.oozie_libext_customer_dir))
shutil.copy2(file, params.oozie_libext_customer_dir)
files_copied = True
if not files_copied:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy ext ZIP to customer dir
oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
if not os.path.isfile(oozie_ext_zip_file):
raise Fail("Unable to copy {0} because it does not exist".format(
oozie_ext_zip_file))
Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file,
params.oozie_libext_customer_dir))
shutil.copy2(oozie_ext_zip_file, params.oozie_libext_customer_dir)
# Redownload jdbc driver to a new current location
if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \
params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver":
File(
params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source),
)
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
params.target),
#creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.target, owner=params.oozie_user, group=params.user_group)
def storm(name=None):
import params
Directory(params.log_dir,
owner=params.storm_user,
group=params.user_group,
mode=0777,
recursive=True
)
Directory([params.pid_dir, params.local_dir],
owner=params.storm_user,
group=params.user_group,
recursive=True,
cd_access="a",
mode=0755,
)
Directory(params.conf_dir,
group=params.user_group,
recursive=True,
cd_access="a",
)
File(format("{conf_dir}/config.yaml"),
content=Template("config.yaml.j2"),
owner=params.storm_user,
group=params.user_group
)
configurations = params.config['configurations']['storm-site']
File(format("{conf_dir}/storm.yaml"),
content=yaml_config_template(configurations),
owner=params.storm_user,
group=params.user_group
)
if params.has_metric_collector:
File(format("{conf_dir}/storm-metrics2.properties"),
owner=params.storm_user,
group=params.user_group,
content=Template("storm-metrics2.properties.j2")
)
File(format("{conf_dir}/storm-env.sh"),
owner=params.storm_user,
content=InlineTemplate(params.storm_env_sh_template)
)
if params.storm_logs_supported:
Directory(params.log4j_dir,
owner=params.storm_user,
group=params.user_group,
mode=0755,
recursive=True
)
File(format("{log4j_dir}/cluster.xml"),
owner=params.storm_user,
content=InlineTemplate(params.storm_cluster_log4j_content)
)
File(format("{log4j_dir}/worker.xml"),
owner=params.storm_user,
content=InlineTemplate(params.storm_worker_log4j_content)
)
if params.security_enabled:
TemplateConfig(format("{conf_dir}/storm_jaas.conf"),
owner=params.storm_user
)
if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0:
TemplateConfig(format("{conf_dir}/client_jaas.conf"),
owner=params.storm_user
)
minRuid = configurations['_storm.min.ruid'] if configurations.has_key('_storm.min.ruid') else ''
min_user_ruid = int(minRuid) if minRuid.isdigit() else _find_real_user_min_uid()
File(format("{conf_dir}/worker-launcher.cfg"),
content=Template("worker-launcher.cfg.j2", min_user_ruid = min_user_ruid),
owner='root',
group=params.user_group
)
