def pre_rolling_restart(self, env):
import params
env.set_params(params)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
Execute(format("hdp-select set spark-historyserver {version}"))
copy_tarballs_to_hdfs('tez', 'spark-historyserver', params.spark_user, params.hdfs_user, params.user_group)
def start(self, env):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
copy_tarballs_to_hdfs("mapreduce", params.mapred_user, params.hdfs_user, params.user_group)
service("historyserver", action="start", serviceName="mapreduce")
def pre_rolling_restart(self, env):
Logger.info("Executing Rolling Upgrade pre-restart")
import params
env.set_params(params)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
Execute(format("hdp-select set hadoop-mapreduce-historyserver {version}"))
copy_tarballs_to_hdfs('mapreduce', 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
def pre_rolling_restart(self, env):
Logger.info("Executing Rolling Upgrade pre-restart")
import params
env.set_params(params)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
Execute(format("hdp-select set hadoop-mapreduce-historyserver {version}"))
copy_tarballs_to_hdfs('mapreduce', 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
def start(self, env, rolling_restart=False):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
copy_tarballs_to_hdfs('mapreduce', 'hadoop-mapreduce-historyserver',
params.mapred_user, params.hdfs_user,
params.user_group)
service('historyserver', action='start', serviceName='mapreduce')
def pre_rolling_restart(self, env):
Logger.info("Executing HiveServer2 Rolling Upgrade pre-restart")
import params
env.set_params(params)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
Execute(format("hdp-select set hive-server2 {version}"))
copy_tarballs_to_hdfs('mapreduce', 'hive-server2', params.tez_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs('tez', 'hive-server2', params.tez_user, params.hdfs_user, params.user_group)
def pre_rolling_restart(self, env):
Logger.info("Executing HiveServer2 Rolling Upgrade pre-restart")
import params
env.set_params(params)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
Execute(format("hdp-select set hive-server2 {version}"))
copy_tarballs_to_hdfs('mapreduce', 'hive-server2', params.tez_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs('tez', 'hive-server2', params.tez_user, params.hdfs_user, params.user_group)
def pre_rolling_restart(self, env):
import params
env.set_params(params)
if params.version and compare_versions(
format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
Execute(format("hdp-select set spark-historyserver {version}"))
copy_tarballs_to_hdfs('tez', 'spark-historyserver',
params.spark_user, params.hdfs_user,
params.user_group)
def start(self, env, rolling_restart=False):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
# This function is needed in HDP 2.2, but it is safe to call in earlier versions.
copy_tarballs_to_hdfs('mapreduce', 'hive-server2', params.tez_user, params.hdfs_user, params.user_group)
setup_ranger_hive()
hive_service( 'hiveserver2', action = 'start',
rolling_restart=rolling_restart )
def start(self, env, rolling_restart=False):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.1') == 0:
install_tez_jars()
else:
# will work only for stack versions >=2.2
copy_tarballs_to_hdfs('tez', 'hadoop-yarn-resourcemanager', params.tez_user, params.hdfs_user, params.user_group)
service('resourcemanager', action='start')
def start(self, env):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
# This function is needed in HDP 2.2, but it is safe to call in earlier versions.
copy_tarballs_to_hdfs('mapreduce', params.tez_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs('tez', params.tez_user, params.hdfs_user, params.user_group)
hive_service( 'hiveserver2',
action = 'start'
)
def start(self, env, rolling_restart=False):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
# This function is needed in HDP 2.2, but it is safe to call in earlier versions.
copy_tarballs_to_hdfs('mapreduce', 'hive-server2', params.tez_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs('tez', 'hive-server2', params.tez_user, params.hdfs_user, params.user_group)
setup_ranger_hive()
hive_service( 'hiveserver2', action = 'start',
rolling_restart=rolling_restart )
self._createHiveDatabase(env)
Links(params.new_hive_log_path, params.hive_log_path)
def start(self, env, rolling_restart=False):
import params
env.set_params(params)
setup_spark(env, 'server', action='start')
if params.security_enabled:
spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
Execute(spark_kinit_cmd, user=params.spark_user)
copy_tarballs_to_hdfs('tez', 'spark-historyserver', params.spark_user, params.hdfs_user, params.user_group)
daemon_cmd = format('{spark_history_server_start}')
no_op_test = format(
'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1')
Execute(daemon_cmd,
user=params.spark_user,
environment={'JAVA_HOME': params.java_home},
not_if=no_op_test
)
def start(self, env, rolling_restart=False):
import params
env.set_params(params)
setup_spark(env, 'server', action='start')
if params.security_enabled:
spark_kinit_cmd = format(
"{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; "
)
Execute(spark_kinit_cmd, user=params.spark_user)
copy_tarballs_to_hdfs('tez', 'spark-historyserver', params.spark_user,
params.hdfs_user, params.user_group)
daemon_cmd = format('{spark_history_server_start}')
no_op_test = format(
'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1'
)
Execute(daemon_cmd,
user=params.spark_user,
environment={'JAVA_HOME': params.java_home},
not_if=no_op_test)
def webhcat():
import params
if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, "2.2.0.0") < 0:
params.HdfsDirectory(params.webhcat_apps_dir,
action="create_delayed",
owner=params.webhcat_user,
mode=0755
)
if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir:
params.HdfsDirectory(params.hcat_hdfs_user_dir,
action="create_delayed",
owner=params.hcat_user,
mode=params.hcat_hdfs_user_mode
)
params.HdfsDirectory(params.webhcat_hdfs_user_dir,
action="create_delayed",
owner=params.webhcat_user,
mode=params.webhcat_hdfs_user_mode
)
params.HdfsDirectory(None, action="create")
Directory(params.templeton_pid_dir,
owner=params.webhcat_user,
mode=0755,
group=params.user_group,
recursive=True)
Directory(params.templeton_log_dir,
owner=params.webhcat_user,
mode=0755,
group=params.user_group,
recursive=True)
Directory(params.config_dir,
recursive=True,
owner=params.webhcat_user,
group=params.user_group)
if params.security_enabled:
kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};")
else:
kinit_if_needed = ""
if kinit_if_needed:
Execute(kinit_if_needed,
user=params.webhcat_user,
path='/bin'
)
# TODO, these checks that are specific to HDP 2.2 and greater should really be in a script specific to that stack.
if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, "2.2.0.0") >= 0:
copy_tarballs_to_hdfs('hive', 'hive-webhcat', params.webhcat_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs('pig', 'hive-webhcat', params.webhcat_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs('hadoop-streaming', 'hive-webhcat', params.webhcat_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs('sqoop', 'hive-webhcat', params.webhcat_user, params.hdfs_user, params.user_group)
else:
CopyFromLocal(params.hadoop_streeming_jars,
owner=params.webhcat_user,
mode=0755,
dest_dir=params.webhcat_apps_dir,
kinnit_if_needed=kinit_if_needed,
hdfs_user=params.hdfs_user,
hadoop_bin_dir=params.hadoop_bin_dir,
hadoop_conf_dir=params.hadoop_conf_dir
)
if (os.path.isfile(params.pig_tar_file)):
CopyFromLocal(params.pig_tar_file,
owner=params.webhcat_user,
mode=0755,
dest_dir=params.webhcat_apps_dir,
kinnit_if_needed=kinit_if_needed,
hdfs_user=params.hdfs_user,
hadoop_bin_dir=params.hadoop_bin_dir,
hadoop_conf_dir=params.hadoop_conf_dir
)
CopyFromLocal(params.hive_tar_file,
owner=params.webhcat_user,
mode=0755,
dest_dir=params.webhcat_apps_dir,
kinnit_if_needed=kinit_if_needed,
hdfs_user=params.hdfs_user,
hadoop_bin_dir=params.hadoop_bin_dir,
hadoop_conf_dir=params.hadoop_conf_dir
)
if (len(glob.glob(params.sqoop_tar_file)) > 0):
CopyFromLocal(params.sqoop_tar_file,
owner=params.webhcat_user,
mode=0755,
dest_dir=params.webhcat_apps_dir,
kinnit_if_needed=kinit_if_needed,
hdfs_user=params.hdfs_user,
hadoop_bin_dir=params.hadoop_bin_dir,
hadoop_conf_dir=params.hadoop_conf_dir
)
# Replace _HOST with hostname in relevant principal-related properties
webhcat_site = params.config['configurations']['webhcat-site'].copy()
for prop_name in ['templeton.hive.properties', 'templeton.kerberos.principal']:
if prop_name in webhcat_site:
webhcat_site[prop_name] = webhcat_site[prop_name].replace("_HOST", params.hostname)
XmlConfig("webhcat-site.xml",
conf_dir=params.config_dir,
configurations=webhcat_site,
configuration_attributes=params.config['configuration_attributes']['webhcat-site'],
owner=params.webhcat_user,
group=params.user_group,
)
File(format("{config_dir}/webhcat-env.sh"),
owner=params.webhcat_user,
group=params.user_group,
content=InlineTemplate(params.webhcat_env_sh_template)
)
Directory(params.webhcat_conf_dir,
cd_access='a',
recursive=True
)
log4j_webhcat_filename = 'webhcat-log4j.properties'
if (params.log4j_webhcat_props != None):
File(format("{config_dir}/{log4j_webhcat_filename}"),
mode=0644,
group=params.user_group,
owner=params.webhcat_user,
content=params.log4j_webhcat_props
)
elif (os.path.exists("{config_dir}/{log4j_webhcat_filename}.template")):
File(format("{config_dir}/{log4j_webhcat_filename}"),
mode=0644,
group=params.user_group,
owner=params.webhcat_user,
content=StaticFile(format("{config_dir}/{log4j_webhcat_filename}.template"))
)
def start(self, env, rolling_restart=False):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
copy_tarballs_to_hdfs('mapreduce', 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
service('historyserver', action='start', serviceName='mapreduce')
def service_check(self, env):
import params
env.set_params(params)
input_file = 'passwd'
output_file = "pigsmoke.out"
cleanup_cmd = format("dfs -rmr {output_file} {input_file}")
#cleanup put below to handle retries; if retrying there wil be a stale file that needs cleanup; exit code is fn of second command
create_file_cmd = format("{cleanup_cmd}; hadoop --config {hadoop_conf_dir} dfs -put /etc/passwd {input_file} ") #TODO: inconsistent that second command needs hadoop
test_cmd = format("fs -test -e {output_file}")
ExecuteHadoop( create_file_cmd,
tries = 3,
try_sleep = 5,
user = params.smokeuser,
conf_dir = params.hadoop_conf_dir,
# for kinit run
keytab = params.smoke_user_keytab,
principal = params.smokeuser_principal,
security_enabled = params.security_enabled,
kinit_path_local = params.kinit_path_local,
bin_dir = params.hadoop_bin_dir
)
File( format("{tmp_dir}/pigSmoke.sh"),
content = StaticFile("pigSmoke.sh"),
mode = 0755
)
# check for Pig-on-M/R
Execute( format("pig {tmp_dir}/pigSmoke.sh"),
tries = 3,
try_sleep = 5,
path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
user = params.smokeuser
)
ExecuteHadoop( test_cmd,
user = params.smokeuser,
conf_dir = params.hadoop_conf_dir,
bin_dir = params.hadoop_bin_dir
)
if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0:
# cleanup results from previous test
ExecuteHadoop( create_file_cmd,
tries = 3,
try_sleep = 5,
user = params.smokeuser,
conf_dir = params.hadoop_conf_dir,
# for kinit run
keytab = params.smoke_user_keytab,
principal = params.smokeuser_principal,
security_enabled = params.security_enabled,
kinit_path_local = params.kinit_path_local,
bin_dir = params.hadoop_bin_dir
)
# Check for Pig-on-Tez
copy_tarballs_to_hdfs('tez', 'hadoop-client', params.smokeuser, params.hdfs_user, params.user_group)
if params.security_enabled:
kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};")
Execute(kinit_cmd,
user=params.smokeuser
)
Execute( format("pig -x tez {tmp_dir}/pigSmoke.sh"),
tries = 3,
try_sleep = 5,
path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
user = params.smokeuser
)
ExecuteHadoop( test_cmd,
user = params.smokeuser,
conf_dir = params.hadoop_conf_dir,
bin_dir = params.hadoop_bin_dir
)
def service_check(self, env):
import params
env.set_params(params)
input_file = 'passwd'
output_file = "pigsmoke.out"
cleanup_cmd = format("dfs -rmr {output_file} {input_file}")
#cleanup put below to handle retries; if retrying there wil be a stale file that needs cleanup; exit code is fn of second command
create_file_cmd = format("{cleanup_cmd}; hadoop --config {hadoop_conf_dir} dfs -put /etc/passwd {input_file} ") #TODO: inconsistent that second command needs hadoop
test_cmd = format("fs -test -e {output_file}")
ExecuteHadoop( create_file_cmd,
tries = 3,
try_sleep = 5,
user = params.smokeuser,
conf_dir = params.hadoop_conf_dir,
# for kinit run
keytab = params.smoke_user_keytab,
security_enabled = params.security_enabled,
kinit_path_local = params.kinit_path_local,
bin_dir = params.hadoop_bin_dir
)
File( format("{tmp_dir}/pigSmoke.sh"),
content = StaticFile("pigSmoke.sh"),
mode = 0755
)
# check for Pig-on-M/R
Execute( format("pig {tmp_dir}/pigSmoke.sh"),
tries = 3,
try_sleep = 5,
path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
user = params.smokeuser
)
ExecuteHadoop( test_cmd,
user = params.smokeuser,
conf_dir = params.hadoop_conf_dir,
bin_dir = params.hadoop_bin_dir
)
if params.stack_is_hdp22_or_further:
# cleanup results from previous test
ExecuteHadoop( create_file_cmd,
tries = 3,
try_sleep = 5,
user = params.smokeuser,
conf_dir = params.hadoop_conf_dir,
# for kinit run
keytab = params.smoke_user_keytab,
security_enabled = params.security_enabled,
kinit_path_local = params.kinit_path_local,
bin_dir = params.hadoop_bin_dir
)
# Check for Pig-on-Tez
copy_tarballs_to_hdfs('tez', params.smokeuser, params.hdfs_user, params.user_group)
if params.security_enabled:
kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser};")
Execute(kinit_cmd,
user=params.smokeuser
)
Execute( format("pig -x tez {tmp_dir}/pigSmoke.sh"),
tries = 3,
try_sleep = 5,
path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
user = params.smokeuser
)
ExecuteHadoop( test_cmd,
user = params.smokeuser,
conf_dir = params.hadoop_conf_dir,
bin_dir = params.hadoop_bin_dir
)
def webhcat():
import params
if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, "2.2.0.0") < 0:
params.HdfsDirectory(params.webhcat_apps_dir,
action="create_delayed",
owner=params.webhcat_user,
mode=0755
)
if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir:
params.HdfsDirectory(params.hcat_hdfs_user_dir,
action="create_delayed",
owner=params.hcat_user,
mode=params.hcat_hdfs_user_mode
)
params.HdfsDirectory(params.webhcat_hdfs_user_dir,
action="create_delayed",
owner=params.webhcat_user,
mode=params.webhcat_hdfs_user_mode
)
params.HdfsDirectory(None, action="create")
Directory(params.templeton_pid_dir,
owner=params.webhcat_user,
mode=0755,
group=params.user_group,
recursive=True)
Directory(params.templeton_log_dir,
owner=params.webhcat_user,
mode=0755,
group=params.user_group,
recursive=True)
Directory(params.config_dir,
recursive=True,
owner=params.webhcat_user,
group=params.user_group)
if params.security_enabled:
kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};")
else:
kinit_if_needed = ""
if kinit_if_needed:
Execute(kinit_if_needed,
user=params.webhcat_user,
path='/bin'
)
# TODO, these checks that are specific to HDP 2.2 and greater should really be in a script specific to that stack.
if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, "2.2.0.0") >= 0:
copy_tarballs_to_hdfs('hive', params.webhcat_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs('pig', params.webhcat_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs('hadoop-streaming', params.webhcat_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs('sqoop', params.webhcat_user, params.hdfs_user, params.user_group)
else:
CopyFromLocal(params.hadoop_streeming_jars,
owner=params.webhcat_user,
mode=0755,
dest_dir=params.webhcat_apps_dir,
kinnit_if_needed=kinit_if_needed,
hdfs_user=params.hdfs_user,
hadoop_bin_dir=params.hadoop_bin_dir,
hadoop_conf_dir=params.hadoop_conf_dir
)
if (os.path.isfile(params.pig_tar_file)):
CopyFromLocal(params.pig_tar_file,
owner=params.webhcat_user,
mode=0755,
dest_dir=params.webhcat_apps_dir,
kinnit_if_needed=kinit_if_needed,
hdfs_user=params.hdfs_user,
hadoop_bin_dir=params.hadoop_bin_dir,
hadoop_conf_dir=params.hadoop_conf_dir
)
CopyFromLocal(params.hive_tar_file,
owner=params.webhcat_user,
mode=0755,
dest_dir=params.webhcat_apps_dir,
kinnit_if_needed=kinit_if_needed,
hdfs_user=params.hdfs_user,
hadoop_bin_dir=params.hadoop_bin_dir,
hadoop_conf_dir=params.hadoop_conf_dir
)
if (len(glob.glob(params.sqoop_tar_file)) > 0):
CopyFromLocal(params.sqoop_tar_file,
owner=params.webhcat_user,
mode=0755,
dest_dir=params.webhcat_apps_dir,
kinnit_if_needed=kinit_if_needed,
hdfs_user=params.hdfs_user,
hadoop_bin_dir=params.hadoop_bin_dir,
hadoop_conf_dir=params.hadoop_conf_dir
)
XmlConfig("webhcat-site.xml",
conf_dir=params.config_dir,
configurations=params.config['configurations']['webhcat-site'],
configuration_attributes=params.config['configuration_attributes']['webhcat-site'],
owner=params.webhcat_user,
group=params.user_group,
)
File(format("{config_dir}/webhcat-env.sh"),
owner=params.webhcat_user,
group=params.user_group,
content=InlineTemplate(params.webhcat_env_sh_template)
)
def webhcat():
import params
if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, "2.2.0.0") < 0:
params.HdfsDirectory(params.webhcat_apps_dir,
action="create_delayed",
owner=params.webhcat_user,
mode=0755
)
if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir:
params.HdfsDirectory(params.hcat_hdfs_user_dir,
action="create_delayed",
owner=params.hcat_user,
mode=params.hcat_hdfs_user_mode
)
params.HdfsDirectory(params.webhcat_hdfs_user_dir,
action="create_delayed",
owner=params.webhcat_user,
mode=params.webhcat_hdfs_user_mode
)
params.HdfsDirectory(None, action="create")
Directory(params.templeton_pid_dir,
owner=params.webhcat_user,
mode=0755,
group=params.user_group,
recursive=True)
Directory(params.templeton_log_dir,
owner=params.webhcat_user,
mode=0755,
group=params.user_group,
recursive=True)
Directory(params.config_dir,
recursive=True,
owner=params.webhcat_user,
group=params.user_group)
if params.security_enabled:
kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};")
else:
kinit_if_needed = ""
if kinit_if_needed:
Execute(kinit_if_needed,
user=params.webhcat_user,
path='/bin'
)
# TODO, these checks that are specific to HDP 2.2 and greater should really be in a script specific to that stack.
if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, "2.2.0.0") >= 0:
copy_tarballs_to_hdfs('hive', 'hive-webhcat', params.webhcat_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs('pig', 'hive-webhcat', params.webhcat_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs('hadoop-streaming', 'hive-webhcat', params.webhcat_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs('sqoop', 'hive-webhcat', params.webhcat_user, params.hdfs_user, params.user_group)
else:
CopyFromLocal(params.hadoop_streeming_jars,
owner=params.webhcat_user,
mode=0755,
dest_dir=params.webhcat_apps_dir,
kinnit_if_needed=kinit_if_needed,
hdfs_user=params.hdfs_user,
hadoop_bin_dir=params.hadoop_bin_dir,
hadoop_conf_dir=params.hadoop_conf_dir
)
if (os.path.isfile(params.pig_tar_file)):
CopyFromLocal(params.pig_tar_file,
owner=params.webhcat_user,
mode=0755,
dest_dir=params.webhcat_apps_dir,
kinnit_if_needed=kinit_if_needed,
hdfs_user=params.hdfs_user,
hadoop_bin_dir=params.hadoop_bin_dir,
hadoop_conf_dir=params.hadoop_conf_dir
)
CopyFromLocal(params.hive_tar_file,
owner=params.webhcat_user,
mode=0755,
dest_dir=params.webhcat_apps_dir,
kinnit_if_needed=kinit_if_needed,
hdfs_user=params.hdfs_user,
hadoop_bin_dir=params.hadoop_bin_dir,
hadoop_conf_dir=params.hadoop_conf_dir
)
if (len(glob.glob(params.sqoop_tar_file)) > 0):
CopyFromLocal(params.sqoop_tar_file,
owner=params.webhcat_user,
mode=0755,
dest_dir=params.webhcat_apps_dir,
kinnit_if_needed=kinit_if_needed,
hdfs_user=params.hdfs_user,
hadoop_bin_dir=params.hadoop_bin_dir,
hadoop_conf_dir=params.hadoop_conf_dir
)
# Replace _HOST with hostname in relevant principal-related properties
webhcat_site = params.config['configurations']['webhcat-site'].copy()
for prop_name in ['templeton.hive.properties', 'templeton.kerberos.principal']:
if prop_name in webhcat_site:
webhcat_site[prop_name] = webhcat_site[prop_name].replace("_HOST", params.hostname)
XmlConfig("webhcat-site.xml",
conf_dir=params.config_dir,
configurations=webhcat_site,
configuration_attributes=params.config['configuration_attributes']['webhcat-site'],
owner=params.webhcat_user,
group=params.user_group,
)
File(format("{config_dir}/webhcat-env.sh"),
owner=params.webhcat_user,
group=params.user_group,
content=InlineTemplate(params.webhcat_env_sh_template)
)
Directory(params.webhcat_conf_dir,
cd_access='a',
recursive=True
)
log4j_webhcat_filename = 'webhcat-log4j.properties'
if (params.log4j_webhcat_props != None):
File(format("{config_dir}/{log4j_webhcat_filename}"),
mode=0644,
group=params.user_group,
owner=params.webhcat_user,
content=params.log4j_webhcat_props
)
elif (os.path.exists("{config_dir}/{log4j_webhcat_filename}.template")):
File(format("{config_dir}/{log4j_webhcat_filename}"),
mode=0644,
group=params.user_group,
owner=params.webhcat_user,
content=StaticFile(format("{config_dir}/{log4j_webhcat_filename}.template"))
)
