def service_check(self, env):
import params
env.set_params(params)
if params.stack_version_formatted and check_stack_feature(
StackFeature.COPY_TARBALL_TO_HDFS,
params.stack_version_formatted):
copy_to_hdfs("slider",
params.user_group,
params.hdfs_user,
skip=params.sysprep_skip_copy_tarballs_hdfs)
smokeuser_kinit_cmd = format(
"{kinit_path_local} -kt {smokeuser_keytab} {smokeuser_principal};"
) if params.security_enabled else ""
servicecheckcmd = format("{smokeuser_kinit_cmd} {slider_cmd} list")
Execute(
servicecheckcmd,
tries=3,
try_sleep=5,
user=params.smokeuser,
logoutput=True,
)
def pre_upgrade_restart(self, env, upgrade_type=None):
Logger.info("Executing Hive Server Stack Upgrade pre-restart")
import params
env.set_params(params)
if params.version and compare_versions(
format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
conf_select.select(params.stack_name, "hive", params.version)
hdp_select.select("hive-server2", params.version)
# Copy mapreduce.tar.gz and tez.tar.gz to HDFS
resource_created = copy_to_hdfs(
"mapreduce",
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped)
resource_created = copy_to_hdfs(
"tez",
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped) or resource_created
if resource_created:
params.HdfsResource(None, action="execute")
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
if check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS,
params.version_for_stack_feature_checks):
# MC Hammer said, "Can't touch this"
resource_created = copy_to_hdfs(
"mapreduce",
params.user_group,
params.hdfs_user,
skip=params.sysprep_skip_copy_tarballs_hdfs)
resource_created = copy_to_hdfs(
"tez",
params.user_group,
params.hdfs_user,
skip=params.sysprep_skip_copy_tarballs_hdfs
) or resource_created
resource_created = copy_to_hdfs(
"slider",
params.user_group,
params.hdfs_user,
skip=params.sysprep_skip_copy_tarballs_hdfs
) or resource_created
if resource_created:
params.HdfsResource(None, action="execute")
else:
# In stack versions before copy_tarball_to_hdfs support tez.tar.gz was copied to a different folder in HDFS.
install_tez_jars()
service('historyserver', action='start', serviceName='mapreduce')
def service_check(self, env):
import params
env.set_params(params)
path_to_tez_jar = format(params.tez_examples_jar)
wordcount_command = format(
"jar {path_to_tez_jar} orderedwordcount /tmp/tezsmokeinput/sample-tez-test /tmp/tezsmokeoutput/"
)
test_command = format("fs -test -e /tmp/tezsmokeoutput/_SUCCESS")
File(format("{tmp_dir}/sample-tez-test"),
content="foo\nbar\nfoo\nbar\nfoo",
mode=0755)
params.HdfsResource("/tmp/tezsmokeoutput",
action="delete_on_execute",
type="directory")
params.HdfsResource(
"/tmp/tezsmokeinput",
action="create_on_execute",
type="directory",
owner=params.smokeuser,
)
params.HdfsResource(
"/tmp/tezsmokeinput/sample-tez-test",
action="create_on_execute",
type="file",
owner=params.smokeuser,
source=format("{tmp_dir}/sample-tez-test"),
)
if params.stack_version_formatted and compare_versions(
params.stack_version_formatted, '2.2.0.0') >= 0:
copy_to_hdfs("tez",
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped)
params.HdfsResource(None, action="execute")
if params.security_enabled:
kinit_cmd = format(
"{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};"
)
Execute(kinit_cmd, user=params.smokeuser)
ExecuteHadoop(wordcount_command,
tries=3,
try_sleep=5,
user=params.smokeuser,
conf_dir=params.hadoop_conf_dir,
bin_dir=params.hadoop_bin_dir)
ExecuteHadoop(test_command,
tries=10,
try_sleep=6,
user=params.smokeuser,
conf_dir=params.hadoop_conf_dir,
bin_dir=params.hadoop_bin_dir)
def pre_upgrade_restart(self, env, upgrade_type=None):
Logger.info(
"Executing Hive Server Interactive Stack Upgrade pre-restart")
import params
env.set_params(params)
if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE,
params.version):
stack_select.select_packages(params.version)
# Copy hive.tar.gz and tez.tar.gz used by Hive Interactive to HDFS
resource_created = copy_to_hdfs(
"hive",
params.user_group,
params.hdfs_user,
skip=params.sysprep_skip_copy_tarballs_hdfs)
resource_created = copy_to_hdfs(
"tez_hive2",
params.user_group,
params.hdfs_user,
skip=params.sysprep_skip_copy_tarballs_hdfs
) or resource_created
resource_created = copy_to_hdfs(
"yarn",
params.user_group,
params.hdfs_user,
skip=params.sysprep_skip_copy_tarballs_hdfs
) or resource_created
if resource_created:
params.HdfsResource(None, action="execute")
def start(self, env, rolling_restart=False):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
if params.hdp_stack_version_major and compare_versions(
params.hdp_stack_version_major, '2.2.0.0') >= 0:
# MC Hammer said, "Can't touch this"
resource_created = copy_to_hdfs(
"mapreduce",
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped)
resource_created = copy_to_hdfs(
"tez",
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped) or resource_created
if resource_created:
params.HdfsResource(None, action="execute")
else:
# In HDP 2.1, tez.tar.gz was copied to a different folder in HDFS.
install_tez_jars()
service('historyserver', action='start', serviceName='mapreduce')
def pre_upgrade_restart(self, env, upgrade_type=None):
Logger.info("Executing Hive Server Stack Upgrade pre-restart")
import params
env.set_params(params)
if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE,
params.version):
stack_select.select("hive-server2", params.version)
# Copy mapreduce.tar.gz and tez.tar.gz to HDFS
resource_created = copy_to_hdfs(
"mapreduce",
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped)
resource_created = copy_to_hdfs(
"tez",
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped) or resource_created
if resource_created:
params.HdfsResource(None, action="execute")
def service_check(self, env):
import params
env.set_params(params)
path_to_tez_jar = format(params.tez_examples_jar)
wordcount_command = format("jar {path_to_tez_jar} orderedwordcount /tmp/tezsmokeinput/sample-tez-test /tmp/tezsmokeoutput/")
test_command = format("fs -test -e /tmp/tezsmokeoutput/_SUCCESS")
File(format("{tmp_dir}/sample-tez-test"),
content = "foo\nbar\nfoo\nbar\nfoo",
mode = 0755
)
params.HdfsResource("/tmp/tezsmokeoutput",
action = "delete_on_execute",
type = "directory"
)
params.HdfsResource("/tmp/tezsmokeinput",
action = "create_on_execute",
type = "directory",
owner = params.smokeuser,
)
params.HdfsResource("/tmp/tezsmokeinput/sample-tez-test",
action = "create_on_execute",
type = "file",
owner = params.smokeuser,
source = format("{tmp_dir}/sample-tez-test"),
)
if params.hdp_stack_version and compare_versions(params.hdp_stack_version, '2.2.0.0') >= 0:
copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
params.HdfsResource(None, action = "execute")
if params.security_enabled:
kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};")
Execute(kinit_cmd,
user=params.smokeuser
)
ExecuteHadoop(wordcount_command,
tries = 3,
try_sleep = 5,
user = params.smokeuser,
conf_dir = params.hadoop_conf_dir,
bin_dir = params.hadoop_bin_dir
)
ExecuteHadoop(test_command,
tries = 10,
try_sleep = 6,
user = params.smokeuser,
conf_dir = params.hadoop_conf_dir,
bin_dir = params.hadoop_bin_dir
)
def pre_upgrade_restart(self, env, upgrade_type=None):
Logger.info("Executing Stack Upgrade post-restart")
import params
env.set_params(params)
if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
stack_select.select_packages(params.version)
# MC Hammer said, "Can't touch this"
copy_to_hdfs("yarn", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs)
params.HdfsResource(None, action="execute")
def pre_rolling_restart(self, env):
Logger.info("Executing Rolling Upgrade pre-restart")
import params
env.set_params(params)
if params.version and compare_versions(
format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
conf_select.select(params.stack_name, "hadoop", params.version)
hdp_select.select("hadoop-mapreduce-historyserver", params.version)
copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user)
params.HdfsResource(None, action="execute")
def pre_upgrade_restart(self, env, upgrade_type=None):
import params
env.set_params(params)
if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE,
params.version):
stack_select.select_packages(params.version)
copy_to_hdfs("mapreduce",
params.user_group,
params.hdfs_user,
use_upgrading_version_during_upgrade=True)
def start(self, env, rolling_restart=False):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
if params.version and compare_versions(
format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user)
params.HdfsResource(None, action="execute")
service('historyserver', action='start', serviceName='mapreduce')
def pre_rolling_restart(self, env):
Logger.info("Executing Rolling Upgrade pre-restart")
import params
env.set_params(params)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
conf_select.select(params.stack_name, "hadoop", params.version)
hdp_select.select("hadoop-mapreduce-historyserver", params.version)
# MC Hammer said, "Can't touch this"
copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
params.HdfsResource(None, action="execute")
def pre_upgrade_restart(self, env, upgrade_type=None):
Logger.info("Executing Stack Upgrade pre-restart")
import params
env.set_params(params)
if params.version and compare_versions(format_stack_version(params.version), '4.0.0.0') >= 0:
conf_select.select(params.stack_name, "hadoop", params.version)
stack_select.select("hadoop-mapreduce-historyserver", params.version)
# MC Hammer said, "Can't touch this"
copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, skip=params.host_sys_prepped)
copy_to_hdfs("slider", params.user_group, params.hdfs_user, skip=params.host_sys_prepped)
params.HdfsResource(None, action="execute")
def pre_rolling_restart(self, env):
Logger.info("Executing HiveServer2 Rolling Upgrade pre-restart")
import params
env.set_params(params)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
conf_select.select(params.stack_name, "hive", params.version)
hdp_select.select("hive-server2", params.version)
# Copy mapreduce.tar.gz and tez.tar.gz to HDFS
resource_created = copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user)
resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user) or resource_created
if resource_created:
params.HdfsResource(None, action="execute")
def start(self, env, rolling_restart=False):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
if params.hdp_stack_version_major and compare_versions(params.hdp_stack_version_major, '2.2.0.0') >= 0:
# MC Hammer said, "Can't touch this"
resource_created = copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user)
resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user) or resource_created
if resource_created:
params.HdfsResource(None, action="execute")
else:
# In HDP 2.1, tez.tar.gz was copied to a different folder in HDFS.
install_tez_jars()
service('historyserver', action='start', serviceName='mapreduce')
def spark_service(action):
import params
if action == 'start':
if params.security_enabled:
spark_kinit_cmd = format(
"{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; "
)
Execute(spark_kinit_cmd, user=params.spark_user)
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
if params.hdp_stack_version and compare_versions(
params.hdp_stack_version, '2.3.0.0') < 0:
resource_created = copy_to_hdfs("tez", params.user_group,
params.hdfs_user)
if resource_created:
params.HdfsResource(None, action="execute")
no_op_test = format(
'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1'
)
Execute(format('{spark_history_server_start}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home},
not_if=no_op_test)
elif action == 'stop':
Execute(format('{spark_history_server_stop}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home})
File(params.spark_history_server_pid_file, action="delete")
def prepare(self, env):
"""
During the "Upgrade" direction of a Stack Upgrade, it is necessary to ensure that the older tez tarball
has been copied to HDFS. This is an additional check for added robustness.
"""
import params
env.set_params(params)
Logger.info(
"Before starting Stack Upgrade, check if tez tarball has been copied to HDFS."
)
if params.hdp_stack_version and compare_versions(
params.hdp_stack_version, '2.2.0.0') >= 0:
Logger.info(
"Stack version {0} is sufficient to check if need to copy tez.tar.gz to HDFS."
.format(params.hdp_stack_version))
# Force it to copy the current version of the tez tarball, rather than the version the RU will go to.
resource_created = copy_to_hdfs(
"tez",
params.user_group,
params.hdfs_user,
use_ru_version_during_ru=False,
host_sys_prepped=params.host_sys_prepped)
if resource_created:
params.HdfsResource(None, action="execute")
else:
raise Fail("Could not copy tez tarball to HDFS.")
def prepare(self, env):
"""
During the "Upgrade" direction of a Stack Upgrade, it is necessary to ensure that the older tez tarball
has been copied to HDFS. This is an additional check for added robustness.
"""
import params
env.set_params(params)
Logger.info(
"Before starting Stack Upgrade, check if tez tarball has been copied to HDFS."
)
if params.stack_version_formatted and check_stack_feature(
StackFeature.ROLLING_UPGRADE, params.stack_version_formatted):
Logger.info(
"Stack version {0} is sufficient to check if need to copy tez.tar.gz to HDFS."
.format(params.stack_version_formatted))
# Force it to copy the current version of the tez tarball, rather than the version the RU will go to.
resource_created = copy_to_hdfs(
"tez",
params.user_group,
params.hdfs_user,
use_upgrading_version_during_upgrade=False,
skip=params.sysprep_skip_copy_tarballs_hdfs)
if resource_created:
params.HdfsResource(None, action="execute")
else:
raise Fail("Could not copy tez tarball to HDFS.")
def spark_service(action):
import params
if action == 'start':
if params.security_enabled:
spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
Execute(spark_kinit_cmd, user=params.spark_user)
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
if params.hdp_stack_version and compare_versions(params.hdp_stack_version, '2.3.0.0') < 0:
resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user)
if resource_created:
params.HdfsResource(None, action="execute")
no_op_test = format(
'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1')
Execute(format('{spark_history_server_start}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home},
not_if=no_op_test
)
elif action == 'stop':
Execute(format('{spark_history_server_stop}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home}
)
File(params.spark_history_server_pid_file,
action="delete"
)
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
# MC Hammer said, "Can't touch this"
resource_created = copy_to_hdfs("mapreduce",
params.user_group,
params.hdfs_user,
skip=params.host_sys_prepped)
resource_created = copy_to_hdfs(
"slider",
params.user_group,
params.hdfs_user,
skip=params.host_sys_prepped) or resource_created
if resource_created:
params.HdfsResource(None, action="execute")
service('historyserver', action='start', serviceName='mapreduce')
def service_check(self, env):
import params
env.set_params(params)
if Script.is_hdp_stack_greater_or_equal("2.2"):
copy_to_hdfs("slider", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
smokeuser_kinit_cmd = format(
"{kinit_path_local} -kt {smokeuser_keytab} {smokeuser_principal};") if params.security_enabled else ""
servicecheckcmd = format("{smokeuser_kinit_cmd} {slider_cmd} list")
Execute(servicecheckcmd,
tries=3,
try_sleep=5,
user=params.smokeuser,
logoutput=True
)
def service_check(self, env):
import params
env.set_params(params)
if Script.is_stack_greater_or_equal("2.2"):
copy_to_hdfs("slider", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
smokeuser_kinit_cmd = format(
"{kinit_path_local} -kt {smokeuser_keytab} {smokeuser_principal};") if params.security_enabled else ""
servicecheckcmd = format("{smokeuser_kinit_cmd} {slider_cmd} list")
Execute(servicecheckcmd,
tries=3,
try_sleep=5,
user=params.smokeuser,
logoutput=True
)
def spark_service(name, action):
import params
if action == "start":
if params.security_enabled:
spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
Execute(spark_kinit_cmd, user=params.spark_user)
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
if params.hdp_stack_version and compare_versions(params.hdp_stack_version, "2.3.0.0") < 0:
resource_created = copy_to_hdfs(
"tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped
)
if resource_created:
params.HdfsResource(None, action="execute")
if name == "jobhistoryserver":
historyserver_no_op_test = format(
"ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1"
)
Execute(
format("{spark_history_server_start}"),
user=params.spark_user,
environment={"JAVA_HOME": params.java_home},
not_if=historyserver_no_op_test,
)
elif name == "sparkthriftserver":
thriftserver_no_op_test = format(
"ls {spark_thrift_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_thrift_server_pid_file}` >/dev/null 2>&1"
)
Execute(
format("{spark_thrift_server_start} --properties-file {spark_thrift_server_conf_file}"),
user=params.spark_user,
environment={"JAVA_HOME": params.java_home},
not_if=thriftserver_no_op_test,
)
elif action == "stop":
if name == "jobhistoryserver":
Execute(
format("{spark_history_server_stop}"),
user=params.spark_user,
environment={"JAVA_HOME": params.java_home},
)
File(params.spark_history_server_pid_file, action="delete")
elif name == "sparkthriftserver":
Execute(
format("{spark_thrift_server_stop}"),
user=params.spark_user,
environment={"JAVA_HOME": params.java_home},
)
File(params.spark_thrift_server_pid_file, action="delete")
def pre_upgrade_restart(self, env, upgrade_type=None):
Logger.info("Executing Stack Upgrade pre-restart")
import params
env.set_params(params)
if params.version and compare_versions(
format_stack_version(params.version), '4.0.0.0') >= 0:
stack_select.select_packages(params.version)
#Execute(format("iop-select set hadoop-mapreduce-historyserver {version}"))
#copy_tarballs_to_hdfs('mapreduce', 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
# MC Hammer said, "Can't touch this"
copy_to_hdfs("mapreduce",
params.user_group,
params.hdfs_user,
skip=params.host_sys_prepped)
copy_to_hdfs("slider",
params.user_group,
params.hdfs_user,
skip=params.host_sys_prepped)
params.HdfsResource(None, action="execute")
def pre_upgrade_restart(self, env, upgrade_type=None):
Logger.info("Executing Hive Server Stack Upgrade pre-restart")
import params
env.set_params(params)
if False:
# Copy mapreduce.tar.gz and tez.tar.gz to HDFS
resource_created = copy_to_hdfs(
"mapreduce",
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped)
resource_created = copy_to_hdfs(
"tez",
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped) or resource_created
if resource_created:
params.HdfsResource(None, action="execute")
def pre_upgrade_restart(self, env, upgrade_type=None):
Logger.info("Executing HiveServer2 Rolling Upgrade pre-restart")
import params
env.set_params(params)
if params.version and compare_versions(format_stack_version(params.version), '4.0.0.0') >= 0:
conf_select.select(params.stack_name, "hive", params.version)
stack_select.select("hive-server2", params.version)
#Execute(format("stack-select set hive-server2 {version}"))
resource_created = copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user)
if resource_created:
params.HdfsResource(None, action="execute")
def install_tez():
import params
Directory([params.config_dir],
owner=params.tez_user,
group=params.user_group,
mode=0775,
create_parents=True)
if not os.path.exists(Script.get_stack_root() + '/' +
params.version_dir) or not os.path.exists(
params.install_dir):
Execute('rm -rf %s' % Script.get_stack_root() + '/' +
params.version_dir)
Execute('rm -rf %s' % params.install_dir)
Execute('wget ' + params.download_url + ' -O /tmp/' + params.filename,
user=params.tez_user)
Execute('tar -zxf /tmp/' + params.filename + ' -C ' +
Script.get_stack_root())
Execute('ln -s ' + Script.get_stack_root() + '/' + params.version_dir +
' ' + params.install_dir)
Execute(' rm -rf ' + params.install_dir + '/conf')
Execute('ln -s ' + params.config_dir + ' ' + params.install_dir +
'/conf')
Execute("echo 'export PATH=%s/bin:$PATH'>/etc/profile.d/tez.sh" %
params.install_dir)
Execute('chown -R %s:%s %s/%s' %
(params.tez_user, params.user_group, params.stack_root,
params.version_dir))
Execute('chown -R %s:%s %s' %
(params.tez_user, params.user_group, params.install_dir))
Execute('/bin/rm -f /tmp/' + params.filename)
copy_to_hdfs("hive",
params.user_group,
params.tez_user,
custom_source_file=params.stack_root +
'/tez/share/tez.tar.gz',
custom_dest_file='/apps/tez/tez.tar.gz')
params.HdfsResource(None, action="execute")
def configure(self, env):
import params
env.set_params(params)
params.HdfsResource('/apps/hbase/coprocessor',
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.user_group,
change_permissions_for_parents=True,
mode=0777)
coprocessor_path = params.install_dir + '/share/hadoop/yarn/timelineservice/hadoop-yarn-server-timelineservice-hbase-coprocessor*.jar'
coprocessor_path_arr = glob.glob(coprocessor_path)
if len(coprocessor_path) > 0:
coprocessor_path = coprocessor_path_arr.pop()
copy_to_hdfs(
"yarn",
params.user_group,
params.hdfs_user,
custom_source_file=coprocessor_path,
custom_dest_file='/apps/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar')
params.HdfsResource(None, action="execute")
yarn(name='apptimelineserver')
def start(self, env, rolling_restart=False):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
if params.has_ranger_admin and params.is_supported_yarn_ranger:
setup_ranger_yarn() #Ranger Yarn Plugin related calls
if not Script.is_hdp_stack_greater_or_equal("2.2"):
install_tez_jars()
else:
resource_created = copy_to_hdfs("tez", params.user_group,
params.hdfs_user)
if resource_created:
params.HdfsResource(None, action="execute")
service('resourcemanager', action='start')
def pre_rolling_restart(self, env):
import params
env.set_params(params)
if params.version and compare_versions(format_hdp_stack_version(params.version), "2.2.0.0") >= 0:
conf_select.select(params.stack_name, "spark", params.version)
hdp_select.select("spark-historyserver", params.version)
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
if params.version and compare_versions(format_hdp_stack_version(params.version), "2.3.0.0") < 0:
resource_created = copy_to_hdfs(
"tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped
)
if resource_created:
params.HdfsResource(None, action="execute")
def pre_rolling_restart(self, env):
import params
env.set_params(params)
if params.version and compare_versions(
format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
conf_select.select(params.stack_name, "spark", params.version)
hdp_select.select("spark-historyserver", params.version)
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
if compare_versions(format_hdp_stack_version(params.version),
'2.3.0.0') < 0:
resource_created = copy_to_hdfs("tez", params.user_group,
params.hdfs_user)
if resource_created:
params.HdfsResource(None, action="execute")
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
if params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major):
# MC Hammer said, "Can't touch this"
resource_created = copy_to_hdfs(
"yarn",
params.user_group,
params.hdfs_user,
skip=params.sysprep_skip_copy_tarballs_hdfs)
if resource_created:
params.HdfsResource(None, action="execute")
if not params.use_external_hbase and not params.is_hbase_system_service_launch:
hbase(action='start')
service('timelinereader', action='start')
def pre_upgrade_restart(self, env, upgrade_type=None):
Logger.info("Executing Stack Upgrade pre-restart")
import params
env.set_params(params)
if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
conf_select.select(params.stack_name, "hadoop", params.version)
stack_select.select("hadoop-mapreduce-historyserver", params.version)
# MC Hammer said, "Can't touch this"
copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs)
copy_to_hdfs("tez", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs)
copy_to_hdfs("slider", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs)
params.HdfsResource(None, action="execute")
def pre_upgrade_restart(self, env, upgrade_type=None):
import params
env.set_params(params)
if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
Logger.info("Executing Spark2 Job History Server Stack Upgrade pre-restart")
conf_select.select(params.stack_name, "spark2", params.version)
stack_select.select("spark2-historyserver", params.version)
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
if params.version and check_stack_feature(StackFeature.TEZ_FOR_SPARK, params.version):
resource_created = copy_to_hdfs(
"tez",
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped)
if resource_created:
params.HdfsResource(None, action="execute")
def pre_upgrade_restart(self, env, upgrade_type=None):
import params
env.set_params(params)
if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
Logger.info("Executing Spark2 Job History Server Stack Upgrade pre-restart")
conf_select.select(params.stack_name, "spark2", params.version)
stack_select.select("spark2-historyserver", params.version)
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
if params.version and check_stack_feature(StackFeature.TEZ_FOR_SPARK, params.version):
resource_created = copy_to_hdfs(
"tez",
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped)
if resource_created:
params.HdfsResource(None, action="execute")
def pre_upgrade_restart(self, env, upgrade_type=None):
import params
env.set_params(params)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
Logger.info("Executing Spark Job History Server Stack Upgrade pre-restart")
conf_select.select(params.stack_name, "spark", params.version)
hdp_select.select("spark-historyserver", params.version)
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.3.0.0') < 0:
resource_created = copy_to_hdfs(
"tez",
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped)
if resource_created:
params.HdfsResource(None, action="execute")
def pre_upgrade_restart(self, env, upgrade_type=None):
import params
env.set_params(params)
if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE,
params.version):
Logger.info(
"Executing Spark Job History Server Stack Upgrade pre-restart")
# TODO, change to "spark" after RPM switches the name
stack_select.select_packages(params.version)
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
if params.version and check_stack_feature(
StackFeature.TEZ_FOR_SPARK, params.version):
resource_created = copy_to_hdfs(
"tez",
params.user_group,
params.hdfs_user,
skip=params.sysprep_skip_copy_tarballs_hdfs)
if resource_created:
params.HdfsResource(None, action="execute")
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
if params.enable_ranger_yarn and params.is_supported_yarn_ranger:
setup_ranger_yarn() #Ranger Yarn Plugin related calls
# wait for active-dir and done-dir to be created by ATS if needed
if params.has_ats:
Logger.info("Verifying DFS directories where ATS stores time line data for active and completed applications.")
self.wait_for_dfs_directories_created(params.entity_groupfs_store_dir, params.entity_groupfs_active_dir)
if params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major):
# MC Hammer said, "Can't touch this"
resource_created = copy_to_hdfs(
"yarn",
params.user_group,
params.hdfs_user,
skip=params.sysprep_skip_copy_tarballs_hdfs) or resource_created
if resource_created:
params.HdfsResource(None, action="execute")
service('resourcemanager', action='start')
def prepare(self, env):
"""
During the "Upgrade" direction of a Rolling Upgrade, it is necessary to ensure that the older tez tarball
has been copied to HDFS. This is an additional check for added robustness.
"""
import params
env.set_params(params)
Logger.info("Before starting Rolling Upgrade, check if tez tarball has been copied to HDFS.")
if params.hdp_stack_version and compare_versions(params.hdp_stack_version, '2.2.0.0') >= 0:
Logger.info("Stack version {0} is sufficient to check if need to copy tez.tar.gz to HDFS.".format(params.hdp_stack_version))
# Force it to copy the current version of the tez tarball, rather than the version the RU will go to.
resource_created = copy_to_hdfs(
"tez",
params.user_group,
params.hdfs_user,
use_ru_version_during_ru=False,
host_sys_prepped=params.host_sys_prepped)
if resource_created:
params.HdfsResource(None, action="execute")
else:
raise Fail("Could not copy tez tarball to HDFS.")
def spark_service(name, upgrade_type=None, action=None):
import params
if action == 'start':
effective_version = params.version if upgrade_type is not None else params.hdp_stack_version
if effective_version:
effective_version = format_hdp_stack_version(effective_version)
if effective_version and compare_versions(effective_version, '2.4.0.0') >= 0:
# copy spark-hdp-assembly.jar to hdfs
copy_to_hdfs("spark", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
# create spark history directory
params.HdfsResource(params.spark_history_dir,
type="directory",
action="create_on_execute",
owner=params.spark_user,
group=params.user_group,
mode=0777,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
if params.security_enabled:
spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
Execute(spark_kinit_cmd, user=params.spark_user)
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
if params.hdp_stack_version and compare_versions(params.hdp_stack_version, '2.3.0.0') < 0:
resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
if resource_created:
params.HdfsResource(None, action="execute")
if name == 'jobhistoryserver':
historyserver_no_op_test = format(
'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1')
Execute(format('{spark_history_server_start}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home},
not_if=historyserver_no_op_test)
elif name == 'sparkthriftserver':
if params.security_enabled:
hive_principal = params.hive_kerberos_principal.replace('_HOST', socket.getfqdn().lower())
hive_kinit_cmd = format("{kinit_path_local} -kt {hive_kerberos_keytab} {hive_principal}; ")
Execute(hive_kinit_cmd, user=params.hive_user)
thriftserver_no_op_test = format(
'ls {spark_thrift_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_thrift_server_pid_file}` >/dev/null 2>&1')
Execute(format('{spark_thrift_server_start} --properties-file {spark_thrift_server_conf_file} {spark_thrift_cmd_opts_properties}'),
user=params.hive_user,
environment={'JAVA_HOME': params.java_home},
not_if=thriftserver_no_op_test
)
elif action == 'stop':
if name == 'jobhistoryserver':
Execute(format('{spark_history_server_stop}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home}
)
File(params.spark_history_server_pid_file,
action="delete"
)
elif name == 'sparkthriftserver':
Execute(format('{spark_thrift_server_stop}'),
user=params.hive_user,
environment={'JAVA_HOME': params.java_home}
)
File(params.spark_thrift_server_pid_file,
action="delete"
)
def service_check(self, env):
import params
env.set_params(params)
input_file = format('/user/{smokeuser}/passwd')
output_dir = format('/user/{smokeuser}/pigsmoke.out')
params.HdfsResource(output_dir,
type="directory",
action="delete_on_execute",
owner=params.smokeuser,
)
params.HdfsResource(input_file,
type="file",
source="/etc/passwd",
action="create_on_execute",
owner=params.smokeuser,
)
params.HdfsResource(None, action="execute")
File( format("{tmp_dir}/pigSmoke.sh"),
content = StaticFile("pigSmoke.sh"),
mode = 0755
)
# check for Pig-on-M/R
Execute( format("pig {tmp_dir}/pigSmoke.sh"),
tries = 3,
try_sleep = 5,
path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
user = params.smokeuser,
logoutput = True
)
test_cmd = format("fs -test -e {output_dir}")
ExecuteHadoop( test_cmd,
user = params.smokeuser,
conf_dir = params.hadoop_conf_dir,
bin_dir = params.hadoop_bin_dir
)
if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0:
# cleanup results from previous test
params.HdfsResource(output_dir,
type="directory",
action="delete_on_execute",
owner=params.smokeuser,
)
params.HdfsResource(input_file,
type="file",
source="/etc/passwd",
action="create_on_execute",
owner=params.smokeuser,
)
# Check for Pig-on-Tez
resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user)
if resource_created:
params.HdfsResource(None, action="execute")
if params.security_enabled:
kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};")
Execute(kinit_cmd,
user=params.smokeuser
)
Execute(format("pig -x tez {tmp_dir}/pigSmoke.sh"),
tries = 3,
try_sleep = 5,
path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
user = params.smokeuser,
logoutput = True
)
ExecuteHadoop(test_cmd,
user = params.smokeuser,
conf_dir = params.hadoop_conf_dir,
bin_dir = params.hadoop_bin_dir
)
def spark_service(name, upgrade_type=None, action=None):
import params
if action == 'start':
effective_version = params.version if upgrade_type is not None else params.stack_version_formatted
if effective_version:
effective_version = format_stack_version(effective_version)
if effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version):
# create & copy spark2-hdp-yarn-archive.tar.gz to hdfs
source_dir=params.spark_home+"/jars"
tmp_archive_file="/tmp/spark2/spark2-hdp-yarn-archive.tar.gz"
make_tarfile(tmp_archive_file, source_dir)
copy_to_hdfs("spark2", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
# create spark history directory
params.HdfsResource(params.spark_history_dir,
type="directory",
action="create_on_execute",
owner=params.spark_user,
group=params.user_group,
mode=0777,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
if params.security_enabled:
spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
Execute(spark_kinit_cmd, user=params.spark_user)
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
if params.stack_version_formatted and check_stack_feature(StackFeature.TEZ_FOR_SPARK, params.stack_version_formatted):
resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
if resource_created:
params.HdfsResource(None, action="execute")
if name == 'jobhistoryserver':
historyserver_no_op_test = format(
'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1')
try:
Execute(format('{spark_history_server_start}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home},
not_if=historyserver_no_op_test)
except:
show_logs(params.spark_log_dir, user=params.spark_user)
raise
elif name == 'sparkthriftserver':
if params.security_enabled:
hive_principal = params.hive_kerberos_principal.replace('_HOST', socket.getfqdn().lower())
hive_kinit_cmd = format("{kinit_path_local} -kt {hive_kerberos_keytab} {hive_principal}; ")
Execute(hive_kinit_cmd, user=params.hive_user)
thriftserver_no_op_test = format(
'ls {spark_thrift_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_thrift_server_pid_file}` >/dev/null 2>&1')
try:
Execute(format('{spark_thrift_server_start} --properties-file {spark_thrift_server_conf_file} {spark_thrift_cmd_opts_properties}'),
user=params.hive_user,
environment={'JAVA_HOME': params.java_home},
not_if=thriftserver_no_op_test
)
except:
show_logs(params.spark_log_dir, user=params.hive_user)
raise
elif action == 'stop':
if name == 'jobhistoryserver':
try:
Execute(format('{spark_history_server_stop}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home}
)
except:
show_logs(params.spark_log_dir, user=params.spark_user)
raise
File(params.spark_history_server_pid_file,
action="delete"
)
elif name == 'sparkthriftserver':
try:
Execute(format('{spark_thrift_server_stop}'),
user=params.hive_user,
environment={'JAVA_HOME': params.java_home}
)
except:
show_logs(params.spark_log_dir, user=params.hive_user)
raise
File(params.spark_thrift_server_pid_file,
action="delete"
)
def hive(name=None):
import params
if name == 'hiveserver2':
# HDP 2.1.* or lower
if params.hdp_stack_version_major != "" and compare_versions(params.hdp_stack_version_major, "2.2.0.0") < 0:
params.HdfsResource(params.webhcat_apps_dir,
type="directory",
action="create_on_execute",
owner=params.webhcat_user,
mode=0755
)
# Create webhcat dirs.
if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir:
params.HdfsResource(params.hcat_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.hcat_user,
mode=params.hcat_hdfs_user_mode
)
params.HdfsResource(params.webhcat_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.webhcat_user,
mode=params.webhcat_hdfs_user_mode
)
# ****** Begin Copy Tarballs ******
# *********************************
# HDP 2.2 or higher, copy mapreduce.tar.gz to HDFS
if params.hdp_stack_version_major != "" and compare_versions(params.hdp_stack_version_major, '2.2') >= 0:
copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
# Always copy pig.tar.gz and hive.tar.gz using the appropriate mode.
# This can use a different source and dest location to account for both HDP 2.1 and 2.2
copy_to_hdfs("pig",
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=params.pig_tar_source,
custom_dest_file=params.pig_tar_dest_file,
host_sys_prepped=params.host_sys_prepped)
copy_to_hdfs("hive",
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=params.hive_tar_source,
custom_dest_file=params.hive_tar_dest_file,
host_sys_prepped=params.host_sys_prepped)
wildcard_tarballs = ["sqoop", "hadoop_streaming"]
for tarball_name in wildcard_tarballs:
source_file_pattern = eval("params." + tarball_name + "_tar_source")
dest_dir = eval("params." + tarball_name + "_tar_dest_dir")
if source_file_pattern is None or dest_dir is None:
continue
source_files = glob.glob(source_file_pattern) if "*" in source_file_pattern else [source_file_pattern]
for source_file in source_files:
src_filename = os.path.basename(source_file)
dest_file = os.path.join(dest_dir, src_filename)
copy_to_hdfs(tarball_name,
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=source_file,
custom_dest_file=dest_file,
host_sys_prepped=params.host_sys_prepped)
# ******* End Copy Tarballs *******
# *********************************
# Create Hive Metastore Warehouse Dir
params.HdfsResource(params.hive_apps_whs_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=0777
)
# Create Hive User Dir
params.HdfsResource(params.hive_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=params.hive_hdfs_user_mode
)
if not is_empty(params.hive_exec_scratchdir) and not urlparse(params.hive_exec_scratchdir).path.startswith("/tmp"):
params.HdfsResource(params.hive_exec_scratchdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.hdfs_user,
mode=0777) # Hive expects this dir to be writeable by everyone as it is used as a temp dir
params.HdfsResource(None, action="execute")
Directory(params.hive_etc_dir_prefix,
mode=0755
)
# We should change configurations for client as well as for server.
# The reason is that stale-configs are service-level, not component.
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
XmlConfig("hive-site.xml",
conf_dir=params.hive_config_dir,
configurations=params.hive_site_config,
configuration_attributes=params.config['configuration_attributes']['hive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
setup_atlas_hive()
if params.hive_specific_configs_supported and name == 'hiveserver2':
XmlConfig("hiveserver2-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']['hiveserver2-site'],
configuration_attributes=params.config['configuration_attributes']['hiveserver2-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
File(format("{hive_config_dir}/hive-env.sh"),
owner=params.hive_user,
group=params.user_group,
content=InlineTemplate(params.hive_env_sh_template)
)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
recursive=True,
owner='root',
group='root'
)
File(os.path.join(params.limits_conf_dir, 'hive.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hive.conf.j2")
)
if (name == 'metastore' or name == 'hiveserver2') and not os.path.exists(params.target):
jdbc_connector()
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
mode = 0644,
)
if name == 'metastore':
File(params.start_metastore_path,
mode=0755,
content=StaticFile('startMetastore.sh')
)
if params.init_metastore_schema:
create_schema_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; "
"{hive_bin}/schematool -initSchema "
"-dbType {hive_metastore_db_type} "
"-userName {hive_metastore_user_name} "
"-passWord {hive_metastore_user_passwd!p}")
check_schema_created_cmd = as_user(format("export HIVE_CONF_DIR={hive_server_conf_dir} ; "
"{hive_bin}/schematool -info "
"-dbType {hive_metastore_db_type} "
"-userName {hive_metastore_user_name} "
"-passWord {hive_metastore_user_passwd!p}"), params.hive_user)
# HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords.
# Fixing it with the hack below:
quoted_hive_metastore_user_passwd = quote_bash_args(quote_bash_args(params.hive_metastore_user_passwd))
if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \
or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"':
quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[1:-1]
Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(check_schema_created_cmd.replace(
format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING))
Execute(create_schema_cmd,
not_if = check_schema_created_cmd,
user = params.hive_user
)
elif name == 'hiveserver2':
File(params.start_hiveserver2_path,
mode=0755,
content=Template(format('{start_hiveserver2_script}'))
)
if name != "client":
crt_directory(params.hive_pid_dir)
crt_directory(params.hive_log_dir)
crt_directory(params.hive_var_lib)
