def copy_jdbc_connector(stack_version=None):
import params
if params.jdbc_jar_name is None and params.driver_curl_source.endswith("/None"):
error_message = "Error! Sorry, but we can't find jdbc driver related to {0} database to download from {1}. \
Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'".format(params.db_flavor, params.jdk_location)
Logger.error(error_message)
if params.driver_curl_source and not params.driver_curl_source.endswith("/None"):
if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar):
File(params.previous_jdbc_jar, action='delete')
kms_home = params.kms_home
if stack_version is not None:
kms_home = format("{stack_root}/{stack_version}/ranger-kms")
driver_curl_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
mode = 0644
)
Directory(os.path.join(kms_home, 'ews', 'lib'),
mode=0755
)
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
Directory(params.jdbc_libs_dir,
cd_access="a",
create_parents=True)
Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
path=["/bin", "/usr/bin/"])
File(os.path.join(kms_home, 'ews', 'webapp', 'lib', 'sajdbc4.jar'), mode=0644)
else:
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(kms_home, 'ews', 'webapp', 'lib', params.jdbc_jar_name), mode=0644)
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = params.config['configurations']['kms-properties'],
owner = params.kms_user
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{kms_home}/ews/webapp/lib/sajdbc4.jar')},
owner = params.kms_user,
)
else:
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
owner = params.kms_user,
)
def start(self, env, upgrade_type=None):
import params
self.configure(env)
Logger.info("Configured Dirs")
pid = \
get_user_call_output.get_user_call_output(format("cat {dpprofiler_pid_file}"), user=params.dpprofiler_user,
is_checked_call=False)[1]
process_exists = format("ls {dpprofiler_pid_file} && ps -p {pid}")
if params.credential_store_enabled:
if 'hadoop.security.credential.provider.path' in params.dpprofiler_config:
credential_provider_path = params.dpprofiler_config['hadoop.security.credential.provider.path']
credential_provider_src_path = credential_provider_path[len('jceks://file'):]
credential_provider_dest_path = params.dpprofiler_credential_provider_path[len('jceks://file'):]
File(credential_provider_dest_path,
owner=params.dpprofiler_user,
group=params.dpprofiler_group,
mode=0600,
content=StaticFile(credential_provider_src_path)
)
Execute(format(
"hadoop credential create {atlas_password_alias} -provider {dpprofiler_credential_provider_path} -value {atlas_password}"))
File(params.dpprofiler_credential_provider_tmp_path,
owner=params.dpprofiler_user,
group=params.dpprofiler_group,
mode=0644,
content=StaticFile(credential_provider_dest_path)
)
credential_provider_hdfs_src_path = params.dpprofiler_credential_provider_hdfs_path[
len('jceks://hdfs'):]
params.HdfsResource(credential_provider_hdfs_src_path,
action="create_on_execute",
type="file",
source=params.dpprofiler_credential_provider_tmp_path,
owner=params.dpprofiler_user,
mode=0600,
recursive_chown=True,
recursive_chmod=True,
dfs_type=params.default_fs
)
File(params.dpprofiler_credential_provider_tmp_path, action="delete")
if os.path.exists(params.dpprofiler_credential_provider_crc_path):
File(params.dpprofiler_credential_provider_crc_path, action="delete")
else:
Logger.error(
"hadoop.security.credential.provider.path property not found in dpprofiler-env config-type")
Logger.info("Starting profiler agent")
environment_dictionary = {}
environment_dictionary["DPPROFILER_CRYPTO_SECRET"] = params.dpprofiler_crypto_secret
kerberos_props = ''
if params.dpprofiler_secured == "true":
kerberos_props = format(
'-Djava.security.krb5.conf=/etc/krb5.conf -Djavax.security.auth.useSubjectCredsOnly=false -Djava.security.auth.login.config={dpprofiler_conf_dir}/krb5JAASLogin.conf')
Execute(format('rm -f {params.dpprofiler_pid_file}'),
not_if=process_exists
)
if params.patch_mysql_driver:
self.patch_mysql_driver()
self.append_to_classpath(params.dpprofiler_hadoop_conf_dir)
Execute(format(
'nohup {dpprofiler_home}/bin/profiler-agent -Dhttp.port={dpprofiler_http_port} {kerberos_props} > {dpprofiler_log_dir}/profiler_agent.out 2>&1 &'),
user=params.dpprofiler_user,
not_if=process_exists,
environment=environment_dictionary
)
try:
# check if pid file created, else fail the task
Execute(format("(ls {dpprofiler_pid_file})"),
tries=20,
try_sleep=3,
)
except:
show_logs(params.dpprofiler_log_dir, params.dpprofiler_user)
raise
newpid = \
get_user_call_output.get_user_call_output(format("cat {dpprofiler_pid_file}"), user=params.dpprofiler_user,
is_checked_call=False)[1]
Logger.info(format("Process pid is: {newpid}"))
def upgrade_schema(self, env):
"""
Executes the schema upgrade binary. This is its own function because it could
be called as a standalone task from the upgrade pack, but is safe to run it for each
metastore instance.
The metastore schema upgrade requires a database driver library for most
databases. During an upgrade, it's possible that the library is not present,
so this will also attempt to copy/download the appropriate driver.
"""
Logger.info("Upgrading Hive Metastore")
import params
env.set_params(params)
if params.security_enabled:
kinit_command = format(
"{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}; "
)
Execute(kinit_command, user=params.smokeuser)
# ensure that the JDBC drive is present for the schema tool; if it's not
# present, then download it first
if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
source = params.source_jdbc_file
target_directory = format("/usr/iop/{version}/hive/lib")
if not os.path.exists(source):
# download it
jdbc_connector()
Execute(('cp', source, target_directory),
path=["/bin", "/usr/bin/"],
sudo=True)
File(
os.path.join(target_directory, os.path.basename(source)),
mode=0644,
)
# build the schema tool command
binary = format("/usr/iop/{version}/hive/bin/schematool")
# the conf.server directory changed locations
# since the configurations have not been written out yet during an upgrade
# we need to choose the original legacy location
schematool_hive_server_conf_dir = params.hive_server_conf_dir
upgrade_from_version = upgrade_summary.get_source_version(
"HIVE", default_version=params.version_for_stack_feature_checks)
if params.version_for_stack_feature_checks is not None:
if compare_versions(upgrade_from_version, "4.1.0.0") < 0:
schematool_hive_server_conf_dir = LEGACY_HIVE_SERVER_CONF
env_dict = {'HIVE_CONF_DIR': schematool_hive_server_conf_dir}
command = format(
"{binary} -dbType {hive_metastore_db_type} -upgradeSchema")
Execute(command,
user=params.hive_user,
tries=1,
environment=env_dict,
logoutput=True)
def kms():
import params
if params.has_ranger_admin:
Directory(params.kms_conf_dir,
owner=params.kms_user,
group=params.kms_group,
recursive=True)
if params.xa_audit_db_is_enabled:
File(params.downloaded_connector_path,
content=DownloadSource(params.driver_source),
mode=0644)
Execute(('cp', '--remove-destination',
params.downloaded_connector_path, params.driver_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_target, mode=0644)
Directory(os.path.join(params.kms_home, 'ews', 'webapp', 'WEB-INF',
'classes', 'lib'),
mode=0755,
owner=params.kms_user,
group=params.kms_group)
Execute(('cp', format('{kms_home}/ranger-kms-initd'),
'/etc/init.d/ranger-kms'),
not_if=format('ls /etc/init.d/ranger-kms'),
only_if=format('ls {kms_home}/ranger-kms-initd'),
sudo=True)
File('/etc/init.d/ranger-kms', mode=0755)
Execute(('chown', '-R', format('{kms_user}:{kms_group}'),
format('{kms_home}/')),
sudo=True)
Directory(params.kms_log_dir,
owner=params.kms_user,
group=params.kms_group)
Execute(('ln', '-sf', format('{kms_home}/ranger-kms'),
'/usr/bin/ranger-kms'),
not_if=format('ls /usr/bin/ranger-kms'),
only_if=format('ls {kms_home}/ranger-kms'),
sudo=True)
File('/usr/bin/ranger-kms', mode=0755)
Execute(('ln', '-sf', format('{kms_home}/ranger-kms'),
'/usr/bin/ranger-kms-services.sh'),
not_if=format('ls /usr/bin/ranger-kms-services.sh'),
only_if=format('ls {kms_home}/ranger-kms'),
sudo=True)
File('/usr/bin/ranger-kms-services.sh', mode=0755)
Execute(('ln', '-sf', format('{kms_home}/ranger-kms-initd'),
format('{kms_home}/ranger-kms-services.sh')),
not_if=format('ls {kms_home}/ranger-kms-services.sh'),
only_if=format('ls {kms_home}/ranger-kms-initd'),
sudo=True)
File(format('{kms_home}/ranger-kms-services.sh'), mode=0755)
Directory(params.kms_log_dir,
owner=params.kms_user,
group=params.kms_group,
mode=0775)
do_keystore_setup(params.credential_provider_path, params.jdbc_alias,
params.db_password)
do_keystore_setup(params.credential_provider_path,
params.masterkey_alias,
params.kms_master_key_password)
XmlConfig(
"dbks-site.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['dbks-site'],
configuration_attributes=params.config['configuration_attributes']
['dbks-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644)
XmlConfig(
"ranger-kms-site.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-site'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644)
XmlConfig(
"kms-site.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['kms-site'],
configuration_attributes=params.config['configuration_attributes']
['kms-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644)
File(os.path.join(params.kms_conf_dir, "kms-log4j.properties"),
owner=params.kms_user,
group=params.kms_group,
content=params.kms_log4j,
mode=0644)
def setup_kms_db():
import params
if params.has_ranger_admin:
password_validation(params.kms_master_key_password, 'KMS master key')
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source),
mode=0644)
Directory(params.java_share_dir,
mode=0755,
recursive=True,
cd_access="a")
if params.db_flavor.lower() != 'sqla':
Execute(('cp', '--remove-destination',
params.downloaded_custom_connector,
params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_curl_target, mode=0644)
Directory(os.path.join(params.kms_home, 'ews', 'lib'), mode=0755)
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C',
params.tmp_dir),
sudo=True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive,
os.path.join(params.kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
Directory(params.jdbc_libs_dir, cd_access="a", recursive=True)
Execute(as_sudo([
'yes', '|', 'cp', params.libs_path_in_archive,
params.jdbc_libs_dir
],
auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(('cp', '--remove-destination',
params.downloaded_custom_connector,
os.path.join(params.kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(params.kms_home, 'ews', 'webapp', 'lib',
params.jdbc_jar_name),
mode=0644)
ModifyPropertiesFile(
format("/usr/hdp/current/ranger-kms/install.properties"),
properties=params.config['configurations']['kms-properties'],
owner=params.kms_user)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(
format("{kms_home}/install.properties"),
properties={
'SQL_CONNECTOR_JAR':
format('{kms_home}/ews/webapp/lib/{jdbc_jar_name}')
},
owner=params.kms_user,
)
env_dict = {
'RANGER_KMS_HOME': params.kms_home,
'JAVA_HOME': params.java_home
}
if params.db_flavor.lower() == 'sqla':
env_dict = {
'RANGER_KMS_HOME': params.kms_home,
'JAVA_HOME': params.java_home,
'LD_LIBRARY_PATH': params.ld_library_path
}
dba_setup = format('ambari-python-wrap {kms_home}/dba_script.py -q')
db_setup = format('ambari-python-wrap {kms_home}/db_setup.py')
Execute(dba_setup,
environment=env_dict,
logoutput=True,
user=params.kms_user)
Execute(db_setup,
environment=env_dict,
logoutput=True,
user=params.kms_user)
def rebalancehdfs(self, env):
import params
env.set_params(params)
name_node_parameters = json.loads( params.name_node_params )
threshold = name_node_parameters['threshold']
_print("Starting balancer with threshold = %s\n" % threshold)
rebalance_env = {'PATH': params.hadoop_bin_dir}
if params.security_enabled:
# Create the kerberos credentials cache (ccache) file and set it in the environment to use
# when executing HDFS rebalance command. Use the md5 hash of the combination of the principal and keytab file
# to generate a (relatively) unique cache filename so that we can use it as needed.
# TODO: params.tmp_dir=/var/lib/ambari-agent/tmp. However hdfs user doesn't have access to this path.
# TODO: Hence using /tmp
ccache_file_name = "hdfs_rebalance_cc_" + _md5(format("{hdfs_principal_name}|{hdfs_user_keytab}")).hexdigest()
ccache_file_path = os.path.join(tempfile.gettempdir(), ccache_file_name)
rebalance_env['KRB5CCNAME'] = ccache_file_path
# If there are no tickets in the cache or they are expired, perform a kinit, else use what
# is in the cache
klist_cmd = format("{klist_path_local} -s {ccache_file_path}")
kinit_cmd = format("{kinit_path_local} -c {ccache_file_path} -kt {hdfs_user_keytab} {hdfs_principal_name}")
if shell.call(klist_cmd, user=params.hdfs_user)[0] != 0:
Execute(kinit_cmd, user=params.hdfs_user)
def calculateCompletePercent(first, current):
# avoid division by zero
try:
division_result = current.bytesLeftToMove/first.bytesLeftToMove
except ZeroDivisionError:
Logger.warning("Division by zero. Bytes Left To Move = {0}. Return 1.0".format(first.bytesLeftToMove))
return 1.0
return 1.0 - division_result
def startRebalancingProcess(threshold, rebalance_env):
rebalanceCommand = format('hdfs --config {hadoop_conf_dir} balancer -threshold {threshold}')
return as_user(rebalanceCommand, params.hdfs_user, env=rebalance_env)
command = startRebalancingProcess(threshold, rebalance_env)
basedir = os.path.join(env.config.basedir, 'scripts')
if(threshold == 'DEBUG'): #FIXME TODO remove this on PROD
basedir = os.path.join(env.config.basedir, 'scripts', 'balancer-emulator')
command = ['ambari-python-wrap','hdfs-command.py']
_print("Executing command %s\n" % command)
parser = hdfs_rebalance.HdfsParser()
def handle_new_line(line, is_stderr):
if is_stderr:
return
_print('[balancer] %s' % (line))
pl = parser.parseLine(line)
if pl:
res = pl.toJson()
res['completePercent'] = calculateCompletePercent(parser.initialLine, pl)
self.put_structured_out(res)
elif parser.state == 'PROCESS_FINISED' :
_print('[balancer] %s' % ('Process is finished' ))
self.put_structured_out({'completePercent' : 1})
return
if (not hdfs_rebalance.is_balancer_running()):
# As the rebalance may take a long time (haours, days) the process is triggered only
# Tracking the progress based on the command output is no longer supported due to this
Execute(command, wait_for_finish=False)
_print("The rebalance process has been triggered")
else:
_print("There is another balancer running. This means you or another Ambari user may have triggered the "
"operation earlier. The process may take a long time to finish (hours, even days). If the problem persists "
"please consult with the HDFS administrators if they have triggred or killed the operation.")
if params.security_enabled:
# Delete the kerberos credentials cache (ccache) file
File(ccache_file_path,
action = "delete",
)
def oozie(is_server=False, upgrade_type=None):
import params
if is_server:
params.HdfsResource(params.oozie_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.oozie_user,
mode=params.oozie_hdfs_user_mode
)
params.HdfsResource(None, action="execute")
generate_logfeeder_input_config('oozie', Template("input.config-oozie.json.j2", extra_imports=[default]))
Directory(params.conf_dir,
create_parents = True,
owner = params.oozie_user,
group = params.user_group
)
params.oozie_site = update_credential_provider_path(params.oozie_site,
'oozie-site',
os.path.join(params.conf_dir, 'oozie-site.jceks'),
params.oozie_user,
params.user_group,
use_local_jceks=True
)
XmlConfig("oozie-site.xml",
conf_dir = params.conf_dir,
configurations = params.oozie_site,
configuration_attributes=params.config['configurationAttributes']['oozie-site'],
owner = params.oozie_user,
group = params.user_group,
mode = 0664
)
File(format("{conf_dir}/oozie-env.sh"),
owner=params.oozie_user,
content=InlineTemplate(params.oozie_env_sh_template),
group=params.user_group,
)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root'
)
File(os.path.join(params.limits_conf_dir, 'oozie.conf'),
owner='root',
group='root',
mode=0644,
content=Template("oozie.conf.j2")
)
if (params.log4j_props != None):
File(format("{params.conf_dir}/oozie-log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.oozie_user,
content=InlineTemplate(params.log4j_props)
)
elif (os.path.exists(format("{params.conf_dir}/oozie-log4j.properties"))):
File(format("{params.conf_dir}/oozie-log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.oozie_user
)
if params.stack_version_formatted and check_stack_feature(StackFeature.OOZIE_ADMIN_USER, params.stack_version_formatted):
File(format("{params.conf_dir}/adminusers.txt"),
mode=0644,
group=params.user_group,
owner=params.oozie_user,
content=Template('adminusers.txt.j2', oozie_admin_users=params.oozie_admin_users)
)
else:
File ( format("{params.conf_dir}/adminusers.txt"),
owner = params.oozie_user,
group = params.user_group
)
if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \
params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
params.jdbc_driver_name == "org.postgresql.Driver" or \
params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver":
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content = DownloadSource(format("{jdk_location}/{check_db_connection_jar_name}")),
)
pass
oozie_ownership()
if params.lzo_enabled:
install_lzo_if_needed()
Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
)
if is_server:
oozie_server_specific(upgrade_type)
def service_check(self, env):
import params
env.set_params(params)
unique = functions.get_unique_id_and_date()
dir = params.hdfs_tmp_dir
tmp_file = format("{dir}/{unique}")
"""
Ignore checking safemode, because this command is unable to get safemode state
when 1 namenode is down in an HA setup (see more in HDFS-8277). Directly
test HDFS availability by file system operations is consistent in both HA and
non-HA environment.
"""
# safemode_command = format("dfsadmin -fs {namenode_address} -safemode get | grep OFF")
if params.security_enabled:
Execute(format(
"{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"
),
user=params.hdfs_user)
#ExecuteHadoop(safemode_command,
# user=params.hdfs_user,
# logoutput=True,
# conf_dir=params.hadoop_conf_dir,
# try_sleep=3,
# tries=20,
# bin_dir=params.hadoop_bin_dir
#)
params.HdfsResource(dir,
type="directory",
action="create_on_execute",
mode=0777)
params.HdfsResource(
tmp_file,
type="file",
action="delete_on_execute",
)
test_file = params.hdfs_service_check_test_file
if not os.path.isfile(test_file):
try:
Execute(format(
"dd if=/dev/urandom of={test_file} count=1 bs=1024"),
user=params.hdfs_user)
except:
File(test_file, action="delete")
test_file = "/etc/passwd"
params.HdfsResource(tmp_file,
type="file",
source=test_file,
action="create_on_execute")
params.HdfsResource(None, action="execute")
if params.has_journalnode_hosts:
if params.security_enabled:
for host in params.journalnode_hosts:
if params.https_only:
uri = format("https://{host}:{journalnode_port}")
else:
uri = format("http://{host}:{journalnode_port}")
response, errmsg, time_millis = curl_krb_request(
params.tmp_dir, params.smoke_user_keytab,
params.smokeuser_principal, uri, "jn_service_check",
params.kinit_path_local, False, None,
params.smoke_user)
if not response:
Logger.error(
"Cannot access WEB UI on: {0}. Error : {1}", uri,
errmsg)
return 1
else:
journalnode_port = params.journalnode_port
checkWebUIFileName = "checkWebUI.py"
checkWebUIFilePath = format("{tmp_dir}/{checkWebUIFileName}")
comma_sep_jn_hosts = ",".join(params.journalnode_hosts)
checkWebUICmd = format(
"ambari-python-wrap {checkWebUIFilePath} -m {comma_sep_jn_hosts} -p {journalnode_port} -s {https_only}"
)
File(checkWebUIFilePath,
content=StaticFile(checkWebUIFileName),
mode=0775)
Execute(checkWebUICmd,
logoutput=True,
try_sleep=3,
tries=5,
user=params.smoke_user)
if params.is_namenode_master:
if params.has_zkfc_hosts:
pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}")
pid_file = format("{pid_dir}/hadoop-{hdfs_user}-zkfc.pid")
check_zkfc_process_cmd = as_user(format(
"ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"
),
user=params.hdfs_user)
Execute(check_zkfc_process_cmd,
logoutput=True,
try_sleep=3,
tries=5)
def namenode(action=None,
hdfs_binary=None,
do_format=True,
upgrade_type=None,
upgrade_suspended=False,
env=None):
if action is None:
raise Fail('"action" parameter is required for function namenode().')
if action in ["start", "stop"] and hdfs_binary is None:
raise Fail(
'"hdfs_binary" parameter is required for function namenode().')
if action == "configure":
import params
#we need this directory to be present before any action(HA manual steps for
#additional namenode)
create_name_dirs(params.dfs_name_dir)
# set up failover / secure zookeper ACLs, this feature is supported from HDP 2.6 ownwards
set_up_zkfc_security(params)
elif action == "start":
Logger.info("Called service {0} with upgrade_type: {1}".format(
action, str(upgrade_type)))
setup_ranger_hdfs(upgrade_type=upgrade_type)
import params
File(params.exclude_file_path,
content=Template("exclude_hosts_list.j2"),
owner=params.hdfs_user,
group=params.user_group)
if do_format and not params.hdfs_namenode_format_disabled:
format_namenode()
pass
if params.dfs_ha_enabled and \
params.dfs_ha_namenode_standby is not None and \
params.hostname == params.dfs_ha_namenode_standby:
# if the current host is the standby NameNode in an HA deployment
# run the bootstrap command, to start the NameNode in standby mode
# this requires that the active NameNode is already up and running,
# so this execute should be re-tried upon failure, up to a timeout
success = bootstrap_standby_namenode(params)
if not success:
raise Fail("Could not bootstrap standby namenode")
if upgrade_type == constants.UPGRADE_TYPE_ROLLING and params.dfs_ha_enabled:
# Most likely, ZKFC is up since RU will initiate the failover command. However, if that failed, it would have tried
# to kill ZKFC manually, so we need to start it if not already running.
safe_zkfc_op(action, env)
options = ""
if upgrade_type == constants.UPGRADE_TYPE_ROLLING:
if params.upgrade_direction == Direction.UPGRADE:
options = "-rollingUpgrade started"
elif params.upgrade_direction == Direction.DOWNGRADE:
options = "-rollingUpgrade downgrade"
elif upgrade_type == constants.UPGRADE_TYPE_NON_ROLLING:
is_previous_image_dir = is_previous_fs_image()
Logger.info("Previous file system image dir present is {0}".format(
str(is_previous_image_dir)))
if params.upgrade_direction == Direction.UPGRADE:
options = "-rollingUpgrade started"
elif params.upgrade_direction == Direction.DOWNGRADE:
options = "-rollingUpgrade downgrade"
elif upgrade_type == constants.UPGRADE_TYPE_HOST_ORDERED:
# nothing special to do for HOU - should be very close to a normal restart
pass
elif upgrade_type is None and upgrade_suspended is True:
# the rollingUpgrade flag must be passed in during a suspended upgrade when starting NN
if os.path.exists(
namenode_upgrade.get_upgrade_in_progress_marker()):
options = "-rollingUpgrade started"
else:
Logger.info(
"The NameNode upgrade marker file {0} does not exist, yet an upgrade is currently suspended. "
"Assuming that the upgrade of NameNode has not occurred yet."
.format(namenode_upgrade.get_upgrade_in_progress_marker()))
Logger.info("Options for start command are: {0}".format(options))
service(action="start",
name="namenode",
user=params.hdfs_user,
options=options,
create_pid_dir=True,
create_log_dir=True)
if params.security_enabled:
Execute(format(
"{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"
),
user=params.hdfs_user)
# ___Scenario___________|_Expected safemode state__|_Wait for safemode OFF____|
# no-HA | ON -> OFF | Yes |
# HA and active | ON -> OFF | Yes |
# HA and standby | no change | No |
# RU with HA on active | ON -> OFF | Yes |
# RU with HA on standby | ON -> OFF | Yes |
# EU with HA on active | ON -> OFF | No |
# EU with HA on standby | ON -> OFF | No |
# EU non-HA | ON -> OFF | No |
# because we do things like create directories after starting NN,
# the vast majority of the time this should be True - it should only
# be False if this is HA and we are the Standby NN
ensure_safemode_off = True
# True if this is the only NameNode (non-HA) or if its the Active one in HA
is_active_namenode = True
if params.dfs_ha_enabled:
Logger.info(
"Waiting for the NameNode to broadcast whether it is Active or Standby..."
)
if is_this_namenode_active() is False:
# we are the STANDBY NN
is_active_namenode = False
# we are the STANDBY NN and this restart is not part of an upgrade
if upgrade_type is None:
ensure_safemode_off = False
# During an Express Upgrade, NameNode will not leave SafeMode until the DataNodes are started,
# so always disable the Safemode check
if upgrade_type == constants.UPGRADE_TYPE_NON_ROLLING:
ensure_safemode_off = False
# some informative logging separate from the above logic to keep things a little cleaner
if ensure_safemode_off:
Logger.info(
"Waiting for this NameNode to leave Safemode due to the following conditions: HA: {0}, isActive: {1}, upgradeType: {2}"
.format(params.dfs_ha_enabled, is_active_namenode,
upgrade_type))
else:
Logger.info(
"Skipping Safemode check due to the following conditions: HA: {0}, isActive: {1}, upgradeType: {2}"
.format(params.dfs_ha_enabled, is_active_namenode,
upgrade_type))
# wait for Safemode to end
if ensure_safemode_off:
if params.rolling_restart and params.rolling_restart_safemode_exit_timeout:
calculated_retries = int(
params.rolling_restart_safemode_exit_timeout) / 30
wait_for_safemode_off(hdfs_binary,
afterwait_sleep=30,
retries=calculated_retries,
sleep_seconds=30)
else:
wait_for_safemode_off(hdfs_binary)
# Always run this on the "Active" NN unless Safemode has been ignored
# in the case where safemode was ignored (like during an express upgrade), then
# NN will be in SafeMode and cannot have directories created
if is_active_namenode and ensure_safemode_off:
create_hdfs_directories()
create_ranger_audit_hdfs_directories()
else:
Logger.info(
"Skipping creation of HDFS directories since this is either not the Active NameNode or we did not wait for Safemode to finish."
)
elif action == "stop":
import params
service(action="stop", name="namenode", user=params.hdfs_user)
elif action == "status":
import status_params
check_process_status(status_params.namenode_pid_file)
elif action == "decommission":
decommission()
def webhcat():
import params
Directory(params.templeton_pid_dir,
owner=params.webhcat_user,
mode=0755,
group=params.user_group,
create_parents=True)
Directory(params.templeton_log_dir,
owner=params.webhcat_user,
mode=0755,
group=params.user_group,
create_parents=True)
Directory(params.config_dir,
create_parents=True,
owner=params.webhcat_user,
group=params.user_group,
cd_access="a")
# Replace _HOST with hostname in relevant principal-related properties
webhcat_site = params.config['configurations']['webhcat-site'].copy()
for prop_name in [
'templeton.hive.properties', 'templeton.kerberos.principal'
]:
if prop_name in webhcat_site:
webhcat_site[prop_name] = webhcat_site[prop_name].replace(
"_HOST", params.hostname)
XmlConfig(
"webhcat-site.xml",
conf_dir=params.config_dir,
configurations=webhcat_site,
configuration_attributes=params.config['configuration_attributes']
['webhcat-site'],
owner=params.webhcat_user,
group=params.user_group,
)
# if we're in an upgrade of a secure cluster, make sure hive-site and yarn-site are created
if params.stack_version_formatted_major and check_stack_feature(StackFeature.CONFIG_VERSIONING, params.stack_version_formatted_major) and \
params.version and params.stack_root:
XmlConfig(
"hive-site.xml",
conf_dir=format("{stack_root}/{version}/hive/conf"),
configurations=params.config['configurations']['hive-site'],
configuration_attributes=params.config['configuration_attributes']
['hive-site'],
owner=params.hive_user,
group=params.user_group,
)
XmlConfig(
"yarn-site.xml",
conf_dir=format("{stack_root}/{version}/hadoop/conf"),
configurations=params.config['configurations']['yarn-site'],
configuration_attributes=params.config['configuration_attributes']
['yarn-site'],
owner=params.yarn_user,
group=params.user_group,
)
File(format("{config_dir}/webhcat-env.sh"),
owner=params.webhcat_user,
group=params.user_group,
content=InlineTemplate(params.webhcat_env_sh_template))
Directory(params.webhcat_conf_dir, cd_access='a', create_parents=True)
log4j_webhcat_filename = 'webhcat-log4j.properties'
if (params.log4j_webhcat_props != None):
File(format("{config_dir}/{log4j_webhcat_filename}"),
mode=0644,
group=params.user_group,
owner=params.webhcat_user,
content=InlineTemplate(params.log4j_webhcat_props))
elif (os.path.exists("{config_dir}/{log4j_webhcat_filename}.template")):
File(format("{config_dir}/{log4j_webhcat_filename}"),
mode=0644,
group=params.user_group,
owner=params.webhcat_user,
content=StaticFile(
format("{config_dir}/{log4j_webhcat_filename}.template")))
# Generate atlas-application.properties.xml file
if params.enable_atlas_hook:
# WebHCat uses a different config dir than the rest of the daemons in Hive.
atlas_hook_filepath = os.path.join(params.config_dir,
params.atlas_hook_filename)
setup_atlas_hook(SERVICE.HIVE,
params.hive_atlas_application_properties,
atlas_hook_filepath, params.hive_user,
params.user_group)
def hdfs(name=None):
import params
if params.create_lib_snappy_symlinks:
install_snappy()
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents = True,
owner='root',
group='root'
)
File(os.path.join(params.limits_conf_dir, 'hdfs.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hdfs.conf.j2")
)
if params.security_enabled:
File(os.path.join(params.hadoop_conf_dir, 'hdfs_dn_jaas.conf'),
owner=params.hdfs_user,
group=params.user_group,
content=Template("hdfs_dn_jaas.conf.j2")
)
File(os.path.join(params.hadoop_conf_dir, 'hdfs_nn_jaas.conf'),
owner=params.hdfs_user,
group=params.user_group,
content=Template("hdfs_nn_jaas.conf.j2")
)
if params.dfs_ha_enabled:
File(os.path.join(params.hadoop_conf_dir, 'hdfs_jn_jaas.conf'),
owner=params.hdfs_user,
group=params.user_group,
content=Template("hdfs_jn_jaas.conf.j2")
)
tc_mode = 0644
tc_owner = "root"
else:
tc_mode = None
tc_owner = params.hdfs_user
if "hadoop-policy" in params.config['configurations']:
XmlConfig("hadoop-policy.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['hadoop-policy'],
configuration_attributes=params.config['configuration_attributes']['hadoop-policy'],
owner=params.hdfs_user,
group=params.user_group
)
if "ssl-client" in params.config['configurations']:
XmlConfig("ssl-client.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configuration_attributes']['ssl-client'],
owner=params.hdfs_user,
group=params.user_group
)
Directory(params.hadoop_conf_secure_dir,
create_parents = True,
owner='root',
group=params.user_group,
cd_access='a',
)
XmlConfig("ssl-client.xml",
conf_dir=params.hadoop_conf_secure_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configuration_attributes']['ssl-client'],
owner=params.hdfs_user,
group=params.user_group
)
if "ssl-server" in params.config['configurations']:
XmlConfig("ssl-server.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['ssl-server'],
configuration_attributes=params.config['configuration_attributes']['ssl-server'],
owner=params.hdfs_user,
group=params.user_group
)
XmlConfig("hdfs-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
owner=params.hdfs_user,
group=params.user_group
)
XmlConfig("core-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configuration_attributes']['core-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644
)
File(os.path.join(params.hadoop_conf_dir, 'slaves'),
owner=tc_owner,
content=Template("slaves.j2")
)
if params.lzo_enabled:
lzo_packages = get_lzo_packages(params.stack_version_unformatted)
Package(lzo_packages,
retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
def oozie(is_server=False):
import params
if is_server:
params.HdfsResource(params.oozie_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.oozie_user,
mode=params.oozie_hdfs_user_mode
)
params.HdfsResource(None, action="execute")
Directory(params.conf_dir,
recursive = True,
owner = params.oozie_user,
group = params.user_group
)
XmlConfig("oozie-site.xml",
conf_dir = params.conf_dir,
configurations = params.oozie_site,
configuration_attributes=params.config['configuration_attributes']['oozie-site'],
owner = params.oozie_user,
group = params.user_group,
mode = 0664
)
File(format("{conf_dir}/oozie-env.sh"),
owner=params.oozie_user,
content=InlineTemplate(params.oozie_env_sh_template)
)
if (params.log4j_props != None):
File(format("{params.conf_dir}/oozie-log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.oozie_user,
content=params.log4j_props
)
elif (os.path.exists(format("{params.conf_dir}/oozie-log4j.properties"))):
File(format("{params.conf_dir}/oozie-log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.oozie_user
)
if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0:
File(format("{params.conf_dir}/adminusers.txt"),
mode=0644,
group=params.user_group,
owner=params.oozie_user,
content=Template('adminusers.txt.j2', oozie_admin_users=params.oozie_admin_users)
)
else:
File ( format("{params.conf_dir}/adminusers.txt"),
owner = params.oozie_user,
group = params.user_group
)
if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \
params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
params.jdbc_driver_name == "org.postgresql.Driver" or \
params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver":
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
)
pass
oozie_ownership()
if is_server:
oozie_server_specific()
def oozie_server_specific():
import params
no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user)
File(params.pid_file,
action="delete",
not_if=no_op_test
)
oozie_server_directories = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir]
Directory( oozie_server_directories,
owner = params.oozie_user,
group = params.user_group,
mode = 0755,
recursive = True,
cd_access="a",
)
Directory(params.oozie_libext_dir,
recursive=True,
)
hashcode_file = format("{oozie_home}/.hashcode")
hashcode = hashlib.md5(format('{oozie_home}/oozie-sharelib.tar.gz')).hexdigest()
skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share && [[ `cat {hashcode_file}` == '{hashcode}' ]]")
untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home)
Execute( untar_sharelib, # time-expensive
not_if = format("{no_op_test} || {skip_recreate_sharelib}"),
sudo = True,
)
configure_cmds = []
configure_cmds.append(('cp', params.ext_js_path, params.oozie_libext_dir))
configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}')))
configure_cmds.append(('chown', '-RL', format('{oozie_user}:{user_group}'), params.oozie_webapps_conf_dir))
Execute( configure_cmds,
not_if = no_op_test,
sudo = True,
)
# download the database JAR
download_database_library_if_needed()
#falcon el extension
if params.has_falcon_host:
Execute(format('{sudo} cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}'),
not_if = no_op_test)
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'),
not_if = no_op_test)
if params.lzo_enabled and len(params.all_lzo_packages) > 0:
Package(params.all_lzo_packages)
Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
not_if = no_op_test,
)
prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd")
prepare_war_cmd = format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure}")
skip_prepare_war_cmd = format("test -f {prepare_war_cmd_file} && [[ `cat {prepare_war_cmd_file}` == '{prepare_war_cmd}' ]]")
Execute(prepare_war_cmd, # time-expensive
user = params.oozie_user,
not_if = format("{no_op_test} || {skip_recreate_sharelib} && {skip_prepare_war_cmd}")
)
File(hashcode_file,
content = hashcode,
mode = 0644,
)
File(prepare_war_cmd_file,
content = prepare_war_cmd,
mode = 0644,
)
if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0:
# Create hive-site and tez-site configs for oozie
Directory(params.hive_conf_dir,
recursive = True,
owner = params.oozie_user,
group = params.user_group
)
if 'hive-site' in params.config['configurations']:
XmlConfig("hive-site.xml",
conf_dir=params.hive_conf_dir,
configurations=params.config['configurations']['hive-site'],
configuration_attributes=params.config['configuration_attributes']['hive-site'],
owner=params.oozie_user,
group=params.user_group,
mode=0644
)
if 'tez-site' in params.config['configurations']:
XmlConfig( "tez-site.xml",
conf_dir = params.hive_conf_dir,
configurations = params.config['configurations']['tez-site'],
configuration_attributes=params.config['configuration_attributes']['tez-site'],
owner = params.oozie_user,
group = params.user_group,
mode = 0664
)
Execute(('chown', '-R', format("{oozie_user}:{user_group}"), params.oozie_server_dir),
sudo=True
)
def enable_kms_plugin():
import params
if params.has_ranger_admin:
ranger_flag = False
if params.stack_supports_ranger_kerberos and params.security_enabled:
if not is_empty(params.rangerkms_principal) and params.rangerkms_principal != '':
ranger_flag = check_ranger_service_support_kerberos(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal)
else:
ranger_flag = check_ranger_service_support_kerberos(params.kms_user, params.spengo_keytab, params.spnego_principal)
else:
ranger_flag = check_ranger_service()
if not ranger_flag:
Logger.error('Error in Get/Create service for Ranger Kms.')
current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
File(format('{kms_conf_dir}/ranger-security.xml'),
owner = params.kms_user,
group = params.kms_group,
mode = 0644,
content = format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>')
)
Directory([os.path.join('/etc', 'ranger', params.repo_name), os.path.join('/etc', 'ranger', params.repo_name, 'policycache')],
owner = params.kms_user,
group = params.kms_group,
mode=0775,
create_parents = True
)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',format('kms_{repo_name}.json')),
owner = params.kms_user,
group = params.kms_group,
mode = 0644
)
# remove plain-text password from xml configs
plugin_audit_properties_copy = {}
plugin_audit_properties_copy.update(params.config['configurations']['ranger-kms-audit'])
if params.plugin_audit_password_property in plugin_audit_properties_copy:
plugin_audit_properties_copy[params.plugin_audit_password_property] = "crypted"
XmlConfig("ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=plugin_audit_properties_copy,
configuration_attributes=params.config['configuration_attributes']['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig("ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-security'],
configuration_attributes=params.config['configuration_attributes']['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
# remove plain-text password from xml configs
ranger_kms_policymgr_ssl_copy = {}
ranger_kms_policymgr_ssl_copy.update(params.config['configurations']['ranger-kms-policymgr-ssl'])
for prop in params.kms_plugin_password_properties:
if prop in ranger_kms_policymgr_ssl_copy:
ranger_kms_policymgr_ssl_copy[prop] = "crypted"
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=ranger_kms_policymgr_ssl_copy,
configuration_attributes=params.config['configuration_attributes']['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'auditDBCred', '-v', PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslKeyStore', '-v', PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslTrustStore', '-v', PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
File(params.credential_file,
owner = params.kms_user,
group = params.kms_group,
mode = 0640
)
# create ranger kms audit directory
if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True
)
params.HdfsResource("/ranger/audit/kms",
type="directory",
action="create_on_execute",
owner=params.kms_user,
group=params.kms_group,
mode=0750,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
if params.xa_audit_hdfs_is_enabled and len(params.namenode_host) > 1:
Logger.info('Audit to Hdfs enabled in NameNode HA environment, creating hdfs-site.xml')
XmlConfig("hdfs-site.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644
)
else:
File(format('{kms_conf_dir}/hdfs-site.xml'), action="delete")
def service(action=None,
name=None,
user=None,
options="",
create_pid_dir=False,
create_log_dir=False):
"""
:param action: Either "start" or "stop"
:param name: Component name, e.g., "namenode", "datanode", "secondarynamenode", "zkfc"
:param user: User to run the command as
:param options: Additional options to pass to command as a string
:param create_pid_dir: Create PID directory
:param create_log_dir: Crate log file directory
"""
import params
options = options if options else ""
pid_dir = format("{hadoop_pid_dir_prefix}/{user}")
pid_file = format("{pid_dir}/hadoop-{user}-{name}.pid")
hadoop_env_exports = {'HADOOP_LIBEXEC_DIR': params.hadoop_libexec_dir}
log_dir = format("{hdfs_log_dir_prefix}/{user}")
# NFS GATEWAY is always started by root using jsvc due to rpcbind bugs
# on Linux such as CentOS6.2. https://bugzilla.redhat.com/show_bug.cgi?id=731542
if name == "nfs3":
pid_file = format("{pid_dir}/hadoop_privileged_nfs3.pid")
custom_export = {
'HADOOP_PRIVILEGED_NFS_USER': params.hdfs_user,
'HADOOP_PRIVILEGED_NFS_PID_DIR': pid_dir,
'HADOOP_PRIVILEGED_NFS_LOG_DIR': log_dir
}
hadoop_env_exports.update(custom_export)
process_id_exists_command = as_sudo(
["test", "-f", pid_file]) + " && " + as_sudo(["pgrep", "-F", pid_file])
# on STOP directories shouldn't be created
# since during stop still old dirs are used (which were created during previous start)
if action != "stop":
if name == "nfs3":
Directory(params.hadoop_pid_dir_prefix,
mode=0755,
owner=params.root_user,
group=params.root_group)
else:
Directory(params.hadoop_pid_dir_prefix,
mode=0755,
owner=params.hdfs_user,
group=params.user_group)
if create_pid_dir:
Directory(pid_dir, owner=user, recursive=True)
if create_log_dir:
if name == "nfs3":
Directory(log_dir,
mode=0775,
owner=params.root_user,
group=params.user_group)
else:
Directory(log_dir, owner=user, recursive=True)
if params.security_enabled and name == "datanode":
## The directory where pid files are stored in the secure data environment.
hadoop_secure_dn_pid_dir = format(
"{hadoop_pid_dir_prefix}/{hdfs_user}")
hadoop_secure_dn_pid_file = format(
"{hadoop_secure_dn_pid_dir}/hadoop_secure_dn.pid")
# At Champlain stack and further, we may start datanode as a non-root even in secure cluster
if not (params.hdp_stack_version != ""
and compare_versions(params.hdp_stack_version, '2.2') >= 0
) or params.secure_dn_ports_are_in_use:
user = "******"
pid_file = format(
"{hadoop_pid_dir_prefix}/{hdfs_user}/hadoop-{hdfs_user}-{name}.pid"
)
if action == 'stop' and (params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0) and \
os.path.isfile(hadoop_secure_dn_pid_file):
# We need special handling for this case to handle the situation
# when we configure non-root secure DN and then restart it
# to handle new configs. Otherwise we will not be able to stop
# a running instance
user = "******"
try:
check_process_status(hadoop_secure_dn_pid_file)
custom_export = {'HADOOP_SECURE_DN_USER': params.hdfs_user}
hadoop_env_exports.update(custom_export)
except ComponentIsNotRunning:
pass
hadoop_daemon = format("{hadoop_bin}/hadoop-daemon.sh")
if user == "root":
cmd = [hadoop_daemon, "--config", params.hadoop_conf_dir, action, name]
if options:
cmd += [
options,
]
daemon_cmd = as_sudo(cmd)
else:
cmd = format(
"{ulimit_cmd} {hadoop_daemon} --config {hadoop_conf_dir} {action} {name}"
)
if options:
cmd += " " + options
daemon_cmd = as_user(cmd, user)
if action == "start":
# remove pid file from dead process
File(pid_file, action="delete", not_if=process_id_exists_command)
Execute(daemon_cmd,
not_if=process_id_exists_command,
environment=hadoop_env_exports)
elif action == "stop":
Execute(daemon_cmd,
only_if=process_id_exists_command,
environment=hadoop_env_exports)
File(pid_file, action="delete")
def storm(name=None):
import params
import os
Directory(
params.log_dir,
owner=params.storm_user,
group=params.user_group,
mode=0777,
create_parents=True,
cd_access="a",
)
Directory(
[params.pid_dir, params.local_dir],
owner=params.storm_user,
group=params.user_group,
create_parents=True,
cd_access="a",
mode=0755,
)
Directory(
params.conf_dir,
group=params.user_group,
create_parents=True,
cd_access="a",
)
File(format("{limits_conf_dir}/storm.conf"),
owner='root',
group='root',
mode=0644,
content=Template("storm.conf.j2"))
File(format("{conf_dir}/config.yaml"),
content=Template("config.yaml.j2"),
owner=params.storm_user,
group=params.user_group)
configurations = params.config['configurations']['storm-site']
File(format("{conf_dir}/storm.yaml"),
content=yaml_config_template(configurations),
owner=params.storm_user,
group=params.user_group)
File(format("{conf_dir}/storm-env.sh"),
owner=params.storm_user,
content=InlineTemplate(params.storm_env_sh_template))
# Generate atlas-application.properties.xml file and symlink the hook jars
if params.enable_atlas_hook:
atlas_hook_filepath = os.path.join(params.conf_dir,
params.atlas_hook_filename)
setup_atlas_hook(SERVICE.STORM,
params.storm_atlas_application_properties,
atlas_hook_filepath, params.storm_user,
params.user_group)
storm_extlib_dir = os.path.join(params.storm_component_home_dir,
"extlib")
setup_atlas_jar_symlinks("storm", storm_extlib_dir)
if params.has_metric_collector:
File(format("{conf_dir}/storm-metrics2.properties"),
owner=params.storm_user,
group=params.user_group,
content=Template("storm-metrics2.properties.j2"))
# Remove symlinks. They can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2
Link(format("{storm_lib_dir}/ambari-metrics-storm-sink.jar"),
action="delete")
# On old HDP 2.1 versions, this symlink may also exist and break EU to newer versions
Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar",
action="delete")
if check_stack_feature(StackFeature.STORM_METRICS_APACHE_CLASSES,
params.version_for_stack_feature_checks):
sink_jar = params.metric_collector_sink_jar
else:
sink_jar = params.metric_collector_legacy_sink_jar
Execute(format(
"{sudo} ln -s {sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar"
),
not_if=format(
"ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"),
only_if=format("ls {sink_jar}"))
if params.storm_logs_supported:
Directory(params.log4j_dir,
owner=params.storm_user,
group=params.user_group,
mode=0755,
create_parents=True)
File(format("{log4j_dir}/cluster.xml"),
owner=params.storm_user,
content=InlineTemplate(params.storm_cluster_log4j_content))
File(format("{log4j_dir}/worker.xml"),
owner=params.storm_user,
content=InlineTemplate(params.storm_worker_log4j_content))
if params.security_enabled:
TemplateConfig(format("{conf_dir}/storm_jaas.conf"),
owner=params.storm_user)
if params.stack_version_formatted and check_stack_feature(
StackFeature.ROLLING_UPGRADE, params.stack_version_formatted):
TemplateConfig(format("{conf_dir}/client_jaas.conf"),
owner=params.storm_user)
minRuid = configurations[
'_storm.min.ruid'] if configurations.has_key(
'_storm.min.ruid') else ''
min_user_ruid = int(
minRuid) if minRuid.isdigit() else _find_real_user_min_uid()
File(format("{conf_dir}/worker-launcher.cfg"),
content=Template("worker-launcher.cfg.j2",
min_user_ruid=min_user_ruid),
owner='root',
group=params.user_group)
def spark_service(name, upgrade_type=None, action=None):
import params
if action == 'start':
effective_version = params.version if upgrade_type is not None else params.stack_version_formatted
if effective_version:
effective_version = format_stack_version(effective_version)
if name == 'jobhistoryserver' and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version):
# create & copy spark-hdp-yarn-archive.tar.gz to hdfs
if not params.sysprep_skip_copy_tarballs_hdfs:
source_dir=params.spark_home+"/jars"
tmp_archive_file=get_tarball_paths("spark")[1]
make_tarfile(tmp_archive_file, source_dir)
copy_to_hdfs("spark", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs, replace_existing_files=True)
# create spark history directory
params.HdfsResource(params.spark_history_dir,
type="directory",
action="create_on_execute",
owner=params.spark_user,
group=params.user_group,
mode=0777,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
if params.security_enabled:
spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
Execute(spark_kinit_cmd, user=params.spark_user)
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
if params.stack_version_formatted and check_stack_feature(StackFeature.TEZ_FOR_SPARK, params.stack_version_formatted):
resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs)
if resource_created:
params.HdfsResource(None, action="execute")
if name == 'jobhistoryserver':
historyserver_no_op_test = format(
'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1')
try:
Execute(format('{spark_history_server_start}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home},
not_if=historyserver_no_op_test)
except:
show_logs(params.spark_log_dir, user=params.spark_user)
raise
elif name == 'sparkthriftserver':
if params.security_enabled:
hive_principal = params.hive_kerberos_principal
hive_kinit_cmd = format("{kinit_path_local} -kt {hive_kerberos_keytab} {hive_principal}; ")
Execute(hive_kinit_cmd, user=params.hive_user)
thriftserver_no_op_test = format(
'ls {spark_thrift_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_thrift_server_pid_file}` >/dev/null 2>&1')
try:
Execute(format('{spark_thrift_server_start} --properties-file {spark_thrift_server_conf_file} {spark_thrift_cmd_opts_properties}'),
user=params.hive_user,
environment={'JAVA_HOME': params.java_home},
not_if=thriftserver_no_op_test
)
except:
show_logs(params.spark_log_dir, user=params.hive_user)
raise
elif action == 'stop':
if name == 'jobhistoryserver':
try:
Execute(format('{spark_history_server_stop}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home}
)
except:
show_logs(params.spark_log_dir, user=params.spark_user)
raise
File(params.spark_history_server_pid_file,
action="delete"
)
elif name == 'sparkthriftserver':
try:
Execute(format('{spark_thrift_server_stop}'),
user=params.hive_user,
environment={'JAVA_HOME': params.java_home}
)
except:
show_logs(params.spark_log_dir, user=params.hive_user)
raise
File(params.spark_thrift_server_pid_file,
action="delete"
)
def hive_service(name, action='start', upgrade_type=None):
import params
if name == 'metastore':
pid_file = format("{hive_pid_dir}/{hive_metastore_pid}")
cmd = format("{start_metastore_path} {hive_log_dir}/hive.out {hive_log_dir}/hive.err {pid_file} {hive_server_conf_dir} {hive_log_dir}")
elif name == 'hiveserver2':
pid_file = format("{hive_pid_dir}/{hive_pid}")
cmd = format("{start_hiveserver2_path} {hive_log_dir}/hive-server2.out {hive_log_dir}/hive-server2.err {pid_file} {hive_server_conf_dir} {hive_log_dir}")
if params.security_enabled and params.current_version and check_stack_feature(StackFeature.HIVE_SERVER2_KERBERIZED_ENV, params.current_version):
hive_kinit_cmd = format("{kinit_path_local} -kt {hive_server2_keytab} {hive_principal}; ")
Execute(hive_kinit_cmd, user=params.hive_user)
pid = get_user_call_output.get_user_call_output(format("cat {pid_file}"), user=params.hive_user, is_checked_call=False)[1]
process_id_exists_command = format("ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1")
if action == 'start':
if name == 'hiveserver2':
check_fs_root(params.hive_server_conf_dir, params.execute_path)
daemon_cmd = cmd
hadoop_home = params.hadoop_home
hive_bin = "hive"
# upgrading hiveserver2 (rolling_restart) means that there is an existing,
# de-registering hiveserver2; the pid will still exist, but the new
# hiveserver is spinning up on a new port, so the pid will be re-written
if upgrade_type == UPGRADE_TYPE_ROLLING:
process_id_exists_command = None
if params.version and params.stack_root:
import os
hadoop_home = format("{stack_root}/{version}/hadoop")
hive_bin = os.path.join(params.hive_bin, hive_bin)
Execute(daemon_cmd,
user = params.hive_user,
environment = { 'HADOOP_HOME': hadoop_home, 'JAVA_HOME': params.java64_home, 'HIVE_BIN': hive_bin },
path = params.execute_path,
not_if = process_id_exists_command)
if params.hive_jdbc_driver == "com.mysql.jdbc.Driver" or \
params.hive_jdbc_driver == "org.postgresql.Driver" or \
params.hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver":
db_connection_check_command = format(
"{java64_home}/bin/java -cp {check_db_connection_jar}:{target_hive} org.apache.ambari.server.DBConnectionVerification '{hive_jdbc_connection_url}' {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}")
try:
Execute(db_connection_check_command,
path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10)
except:
show_logs(params.hive_log_dir, params.hive_user)
raise
elif action == 'stop':
daemon_kill_cmd = format("{sudo} kill {pid}")
daemon_hard_kill_cmd = format("{sudo} kill -9 {pid}")
Execute(daemon_kill_cmd,
not_if = format("! ({process_id_exists_command})")
)
wait_time = 5
Execute(daemon_hard_kill_cmd,
not_if = format("! ({process_id_exists_command}) || ( sleep {wait_time} && ! ({process_id_exists_command}) )")
)
try:
# check if stopped the process, else fail the task
Execute(format("! ({process_id_exists_command})"),
tries=20,
try_sleep=3,
)
except:
show_logs(params.hive_log_dir, params.hive_user)
raise
File(pid_file,
action = "delete"
)
def kafka(upgrade_type=None):
import params
ensure_base_directories()
kafka_server_config = mutable_config_dict(
params.config['configurations']['kafka-broker'])
# This still has an issue of hostnames being alphabetically out-of-order for broker.id in HDP-2.2.
# Starting in HDP 2.3, Kafka handles the generation of broker.id so Ambari doesn't have to.
effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(
params.version)
Logger.info(format("Effective stack version: {effective_version}"))
if effective_version is not None and effective_version != "" and \
check_stack_feature(StackFeature.CREATE_KAFKA_BROKER_ID, effective_version):
if len(params.kafka_hosts
) > 0 and params.hostname in params.kafka_hosts:
brokerid = str(sorted(params.kafka_hosts).index(params.hostname))
kafka_server_config['broker.id'] = brokerid
Logger.info(format("Calculating broker.id as {brokerid}"))
# listeners and advertised.listeners are only added in 2.3.0.0 onwards.
if effective_version is not None and effective_version != "" and \
check_stack_feature(StackFeature.KAFKA_LISTENERS, effective_version):
listeners = kafka_server_config['listeners'].replace(
"localhost", params.hostname)
Logger.info(format("Kafka listeners: {listeners}"))
if params.security_enabled and params.kafka_kerberos_enabled:
Logger.info("Kafka kerberos security is enabled.")
if "SASL" not in listeners:
listeners = listeners.replace("PLAINTEXT", "PLAINTEXTSASL")
kafka_server_config['listeners'] = listeners
kafka_server_config['advertised.listeners'] = listeners
Logger.info(format("Kafka advertised listeners: {listeners}"))
else:
kafka_server_config['listeners'] = listeners
if 'advertised.listeners' in kafka_server_config:
advertised_listeners = kafka_server_config[
'advertised.listeners'].replace("localhost",
params.hostname)
kafka_server_config[
'advertised.listeners'] = advertised_listeners
Logger.info(
format(
"Kafka advertised listeners: {advertised_listeners}"))
else:
kafka_server_config['host.name'] = params.hostname
if params.has_metric_collector:
kafka_server_config[
'kafka.timeline.metrics.host'] = params.metric_collector_host
kafka_server_config[
'kafka.timeline.metrics.port'] = params.metric_collector_port
kafka_server_config[
'kafka.timeline.metrics.protocol'] = params.metric_collector_protocol
kafka_server_config[
'kafka.timeline.metrics.truststore.path'] = params.metric_truststore_path
kafka_server_config[
'kafka.timeline.metrics.truststore.type'] = params.metric_truststore_type
kafka_server_config[
'kafka.timeline.metrics.truststore.password'] = params.metric_truststore_password
kafka_data_dir = kafka_server_config['log.dirs']
kafka_data_dirs = filter(None, kafka_data_dir.split(","))
Directory(
kafka_data_dirs,
mode=0755,
cd_access='a',
owner=params.kafka_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True,
)
PropertiesFile(
"server.properties",
dir=params.conf_dir,
properties=kafka_server_config,
owner=params.kafka_user,
group=params.user_group,
)
File(format("{conf_dir}/kafka-env.sh"),
owner=params.kafka_user,
content=InlineTemplate(params.kafka_env_sh_template))
if (params.log4j_props != None):
File(format("{conf_dir}/log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.kafka_user,
content=params.log4j_props)
if params.security_enabled and params.kafka_kerberos_enabled:
TemplateConfig(format("{conf_dir}/kafka_jaas.conf"),
owner=params.kafka_user)
TemplateConfig(format("{conf_dir}/kafka_client_jaas.conf"),
owner=params.kafka_user)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'kafka.conf'),
owner='root',
group='root',
mode=0644,
content=Template("kafka.conf.j2"))
File(os.path.join(params.conf_dir, 'tools-log4j.properties'),
owner='root',
group='root',
mode=0644,
content=Template("tools-log4j.properties.j2"))
setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir)
setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)
def hive_interactive(name=None):
import params
MB_TO_BYTES = 1048576
# if warehouse directory is in DFS
if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(
params.default_fs).scheme:
# Create Hive Metastore Warehouse Dir
params.HdfsResource(params.hive_metastore_warehouse_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=0700)
# create directories for llap package
pkg_dir = '/user/' + params.hive_user + '/.yarn'
for dir in [pkg_dir, pkg_dir + '/package', pkg_dir + '/package/LLAP']:
# hdfsresouces handles parent creation badly
params.HdfsResource(dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=0755)
if not is_empty(params.tez_hook_proto_base_directory):
params.HdfsResource(params.tez_hook_proto_base_directory,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01755)
if not is_empty(params.hive_hook_proto_base_directory):
params.HdfsResource(params.hive_hook_proto_base_directory,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
dag_meta = params.tez_hook_proto_base_directory + "dag_meta"
params.HdfsResource(dag_meta,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
dag_data = params.tez_hook_proto_base_directory + "dag_data"
params.HdfsResource(dag_data,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
app_data = params.tez_hook_proto_base_directory + "app_data"
params.HdfsResource(app_data,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
else:
Logger.info(
format(
"Not creating warehouse directory '{hive_metastore_warehouse_dir}', as the location is not in DFS."
))
# Create Hive User Dir
params.HdfsResource(params.hive_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=params.hive_hdfs_user_mode)
params.HdfsResource(None, action="execute")
# list of properties that should be excluded from the config
# this approach is a compromise against adding a dedicated config
# type for hive_server_interactive or needed config groups on a
# per component basis
exclude_list = ['hive.enforce.bucketing', 'hive.enforce.sorting']
# List of configs to be excluded from hive2 client, but present in Hive2 server.
exclude_list_for_hive2_client = [
'javax.jdo.option.ConnectionPassword',
'hadoop.security.credential.provider.path'
]
Logger.info("Directories to fill with configs: %s" %
str(params.hive_conf_dirs_list))
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
'''
As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml,
we need to merge hive/hive-site.xml and hive2/hive-site.xml and store it in hive2/hive-site.xml.
'''
merged_hive_interactive_site = {}
merged_hive_interactive_site.update(params.hive_site_config)
merged_hive_interactive_site.update(
params.config['configurations']['hive-interactive-site'])
for item in exclude_list:
if item in merged_hive_interactive_site.keys():
del merged_hive_interactive_site[item]
merged_hive_interactive_site[
'hive.llap.daemon.vcpus.per.instance'] = format(
merged_hive_interactive_site['hive.llap.daemon.vcpus.per.instance']
)
merged_hive_interactive_site[
'hive.server2.active.passive.ha.enable'] = str(
params.hive_server_interactive_ha).lower()
'''
Config 'hive.llap.io.memory.size' calculated value in stack_advisor is in MB as of now. We need to
convert it to bytes before we write it down to config file.
'''
if 'hive.llap.io.memory.size' in merged_hive_interactive_site.keys():
hive_llap_io_mem_size_in_mb = merged_hive_interactive_site.get(
"hive.llap.io.memory.size")
hive_llap_io_mem_size_in_bytes = long(
hive_llap_io_mem_size_in_mb) * MB_TO_BYTES
merged_hive_interactive_site[
'hive.llap.io.memory.size'] = hive_llap_io_mem_size_in_bytes
Logger.info(
"Converted 'hive.llap.io.memory.size' value from '{0} MB' to '{1} Bytes' before writing "
"it to config file.".format(hive_llap_io_mem_size_in_mb,
hive_llap_io_mem_size_in_bytes))
'''
Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook',
which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed.
'''
# Generate atlas-application.properties.xml file
if params.enable_atlas_hook and params.stack_supports_atlas_hook_for_hive_interactive:
Logger.info("Setup for Atlas Hive2 Hook started.")
atlas_hook_filepath = os.path.join(
params.hive_server_interactive_conf_dir,
params.atlas_hook_filename)
setup_atlas_hook(SERVICE.HIVE,
params.hive_atlas_application_properties,
atlas_hook_filepath, params.hive_user,
params.user_group)
Logger.info("Setup for Atlas Hive2 Hook done.")
else:
# Required for HDP 2.5 stacks
Logger.info(
"Skipping setup for Atlas Hook, as it is disabled/ not supported.")
remove_atlas_hook_if_exists(merged_hive_interactive_site)
'''
As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml,
we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml.
'''
merged_tez_interactive_site = {}
if 'tez-site' in params.config['configurations']:
merged_tez_interactive_site.update(
params.config['configurations']['tez-site'])
Logger.info(
"Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'."
)
else:
Logger.error(
"Tez's 'tez-site' couldn't be retrieved from passed-in configurations."
)
merged_tez_interactive_site.update(
params.config['configurations']['tez-interactive-site'])
XmlConfig("tez-site.xml",
conf_dir=params.tez_interactive_conf_dir,
configurations=merged_tez_interactive_site,
configuration_attributes=params.config['configurationAttributes']
['tez-interactive-site'],
owner=params.tez_interactive_user,
group=params.user_group,
mode=0664)
'''
Merge properties from hiveserver2-interactive-site into hiveserver2-site
'''
merged_hiveserver2_interactive_site = {}
if 'hiveserver2-site' in params.config['configurations']:
merged_hiveserver2_interactive_site.update(
params.config['configurations']['hiveserver2-site'])
Logger.info(
"Retrieved 'hiveserver2-site' for merging with 'hiveserver2-interactive-site'."
)
else:
Logger.error(
"'hiveserver2-site' couldn't be retrieved from passed-in configurations."
)
merged_hiveserver2_interactive_site.update(
params.config['configurations']['hiveserver2-interactive-site'])
# Create config files under hive_server_interactive_conf_dir:
# hive-site.xml
# hive-env.sh
# llap-daemon-log4j2.properties
# llap-cli-log4j2.properties
# hive-log4j2.properties
# hive-exec-log4j2.properties
# beeline-log4j2.properties
hive_server_interactive_conf_dir = params.hive_server_interactive_conf_dir
mode_identified = 0600
merged_hive_interactive_site = update_credential_provider_path(
merged_hive_interactive_site, 'hive-site',
os.path.join(conf_dir, 'hive-site.jceks'), params.hive_user,
params.user_group)
XmlConfig("hive-site.xml",
conf_dir=hive_server_interactive_conf_dir,
configurations=merged_hive_interactive_site,
configuration_attributes=params.config['configurationAttributes']
['hive-interactive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
XmlConfig("hiveserver2-site.xml",
conf_dir=hive_server_interactive_conf_dir,
configurations=merged_hiveserver2_interactive_site,
configuration_attributes=params.config['configurationAttributes']
['hiveserver2-interactive-site'],
owner=params.hive_user,
group=params.user_group,
mode=mode_identified)
File(format("{hive_server_interactive_conf_dir}/hive-env.sh"),
owner=params.hive_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.hive_interactive_env_sh_template))
llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.llap_daemon_log4j))
llap_cli_log4j2_filename = 'llap-cli-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.llap_cli_log4j2))
hive_log4j2_filename = 'hive-log4j2.properties'
File(format("{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.hive_log4j2))
hive_exec_log4j2_filename = 'hive-exec-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.hive_exec_log4j2))
beeline_log4j2_filename = 'beeline-log4j2.properties'
File(
format("{hive_server_interactive_conf_dir}/{beeline_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.beeline_log4j2))
XmlConfig("beeline-site.xml",
conf_dir=conf_dir,
configurations=params.beeline_site_config,
owner=params.hive_user,
group=params.user_group,
mode=mode_identified)
File(os.path.join(hive_server_interactive_conf_dir,
"hadoop-metrics2-hiveserver2.properties"),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=Template("hadoop-metrics2-hiveserver2.properties.j2"))
File(format(
"{hive_server_interactive_conf_dir}/hadoop-metrics2-llapdaemon.properties"
),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=Template("hadoop-metrics2-llapdaemon.j2"))
File(format(
"{hive_server_interactive_conf_dir}/hadoop-metrics2-llaptaskscheduler.properties"
),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=Template("hadoop-metrics2-llaptaskscheduler.j2"))
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'hive.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hive.conf.j2"))
if not os.path.exists(params.target_hive_interactive):
jdbc_connector(params.target_hive_interactive,
params.hive_intaractive_previous_jdbc_jar)
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}/{check_db_connection_jar_name}")),
mode=0644)
File(params.start_hiveserver2_interactive_path,
mode=0755,
content=Template(format('{start_hiveserver2_interactive_script}')))
Directory(params.hive_pid_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_log_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_interactive_var_lib,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
generate_logfeeder_input_config(
'hive', Template("input.config-hive.json.j2", extra_imports=[default]))
def oozie_server_specific(upgrade_type):
import params
no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user)
File(params.pid_file,
action="delete",
not_if=no_op_test
)
oozie_server_directories = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir]
Directory( oozie_server_directories,
owner = params.oozie_user,
group = params.user_group,
mode = 0755,
create_parents = True,
cd_access="a",
)
Directory(params.oozie_libext_dir,
create_parents = True,
)
hashcode_file = format("{oozie_home}/.hashcode")
skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share")
untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home)
Execute(('ln','-sf', format('{oozie_webapps_conf_dir}'), format('{oozie_server_dir}')), sudo=True)
Execute( untar_sharelib, # time-expensive
not_if = format("{no_op_test} || {skip_recreate_sharelib}"),
sudo = True,
)
configure_cmds = []
# Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params)
# Copy the first oozie ext-2.2.zip file that is found.
# This uses a list to handle the cases when migrating from some versions of BigInsights to HDP.
if source_ext_zip_paths is not None:
for source_ext_zip_path in source_ext_zip_paths:
if os.path.isfile(source_ext_zip_path):
configure_cmds.append(('cp', source_ext_zip_path, params.oozie_libext_dir))
configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}')))
Execute(configure_cmds,
not_if=no_op_test,
sudo=True,
)
break
Directory(params.oozie_webapps_conf_dir,
owner = params.oozie_user,
group = params.user_group,
recursive_ownership = True,
recursion_follow_links = True,
)
# download the database JAR
download_database_library_if_needed()
#falcon el extension
if params.has_falcon_host:
Execute(format('{sudo} cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}'),
not_if = no_op_test)
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'),
not_if = no_op_test)
prepare_war(params)
File(hashcode_file,
mode = 0644,
)
if params.stack_version_formatted and check_stack_feature(StackFeature.OOZIE_CREATE_HIVE_TEZ_CONFIGS, params.stack_version_formatted):
# Create hive-site and tez-site configs for oozie
Directory(params.hive_conf_dir,
create_parents = True,
owner = params.oozie_user,
group = params.user_group
)
if 'hive-site' in params.config['configurations']:
hive_site_config = update_credential_provider_path(params.config['configurations']['hive-site'],
'hive-site',
os.path.join(params.hive_conf_dir, 'hive-site.jceks'),
params.oozie_user,
params.user_group
)
XmlConfig("hive-site.xml",
conf_dir=params.hive_conf_dir,
configurations=hive_site_config,
configuration_attributes=params.config['configurationAttributes']['hive-site'],
owner=params.oozie_user,
group=params.user_group,
mode=0644
)
if 'tez-site' in params.config['configurations']:
XmlConfig( "tez-site.xml",
conf_dir = params.hive_conf_dir,
configurations = params.config['configurations']['tez-site'],
configuration_attributes=params.config['configurationAttributes']['tez-site'],
owner = params.oozie_user,
group = params.user_group,
mode = 0664
)
# If Atlas is also installed, need to generate Atlas Hive hook (hive-atlas-application.properties file) in directory
# {stack_root}/{current_version}/atlas/hook/hive/
# Because this is a .properties file instead of an xml file, it will not be read automatically by Oozie.
# However, should still save the file on this host so that can upload it to the Oozie Sharelib in DFS.
if has_atlas_in_cluster():
atlas_hook_filepath = os.path.join(params.hive_conf_dir, params.atlas_hook_filename)
Logger.info("Has atlas in cluster, will save Atlas Hive hook into location %s" % str(atlas_hook_filepath))
setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.oozie_user, params.user_group)
Directory(params.oozie_server_dir,
owner = params.oozie_user,
group = params.user_group,
recursive_ownership = True,
)
if params.security_enabled:
File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'),
owner=params.oozie_user,
group=params.user_group,
content=Template("zkmigrator_jaas.conf.j2")
)
def setup_logsearch():
import params
Directory([params.logsearch_log_dir, params.logsearch_pid_dir],
mode=0755,
cd_access='a',
owner=params.logsearch_user,
group=params.user_group,
create_parents=True)
Directory([
params.logsearch_dir, params.logsearch_server_conf,
params.logsearch_config_set_dir
],
mode=0755,
cd_access='a',
owner=params.logsearch_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True)
Directory(params.logsearch_server_keys_folder,
cd_access='a',
mode=0755,
owner=params.logsearch_user,
group=params.user_group)
File(params.logsearch_log,
mode=0644,
owner=params.logsearch_user,
group=params.user_group,
content='')
if params.credential_store_enabled:
params.logsearch_env_config = update_credential_provider_path(
params.logsearch_env_config, 'logsearch-env',
params.logsearch_env_jceks_file, params.logsearch_user,
params.user_group)
params.logsearch_properties[
HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'jceks://file' + params.logsearch_env_jceks_file
File(format("{logsearch_server_keys_folder}/ks_pass.txt"),
action="delete")
File(format("{logsearch_server_keys_folder}/ts_pass.txt"),
action="delete")
else:
File(format("{logsearch_server_keys_folder}/ks_pass.txt"),
content=params.logsearch_keystore_password,
mode=0600,
owner=params.logsearch_user,
group=params.user_group)
File(format("{logsearch_server_keys_folder}/ts_pass.txt"),
content=params.logsearch_truststore_password,
mode=0600,
owner=params.logsearch_user,
group=params.user_group)
PropertiesFile(format("{logsearch_server_conf}/logsearch.properties"),
properties=params.logsearch_properties)
File(format("{logsearch_server_conf}/HadoopServiceConfig.json"),
content=Template("HadoopServiceConfig.json.j2"),
owner=params.logsearch_user,
group=params.user_group)
File(format("{logsearch_server_conf}/log4j.xml"),
content=InlineTemplate(params.logsearch_app_log4j_content),
owner=params.logsearch_user,
group=params.user_group)
File(format("{logsearch_server_conf}/logsearch-env.sh"),
content=InlineTemplate(params.logsearch_env_content),
mode=0755,
owner=params.logsearch_user,
group=params.user_group)
File(format("{logsearch_server_conf}/logsearch-admin.json"),
content=InlineTemplate(params.logsearch_admin_content),
owner=params.logsearch_user,
group=params.user_group)
File(format("{logsearch_config_set_dir}/hadoop_logs/conf/solrconfig.xml"),
content=InlineTemplate(
params.logsearch_service_logs_solrconfig_content),
owner=params.logsearch_user,
group=params.user_group)
File(format("{logsearch_config_set_dir}/audit_logs/conf/solrconfig.xml"),
content=InlineTemplate(
params.logsearch_audit_logs_solrconfig_content),
owner=params.logsearch_user,
group=params.user_group)
if params.security_enabled:
kinit_cmd = format(
"{kinit_path_local} -kt {logsearch_kerberos_keytab} {logsearch_kerberos_principal};"
)
Execute(kinit_cmd, user=params.logsearch_user)
if params.security_enabled:
File(format("{logsearch_jaas_file}"),
content=Template("logsearch_jaas.conf.j2"),
owner=params.logsearch_user)
Execute(("chmod", "-R", "ugo+r",
format("{logsearch_server_conf}/solr_configsets")),
sudo=True)
check_znode()
if params.security_enabled and not params.logsearch_use_external_solr:
solr_cloud_util.add_solr_roles(
params.config,
roles=[
params.infra_solr_role_logsearch,
params.infra_solr_role_ranger_admin, params.infra_solr_role_dev
],
new_service_principals=[params.logsearch_kerberos_principal])
solr_cloud_util.add_solr_roles(
params.config,
roles=[
params.infra_solr_role_logfeeder, params.infra_solr_role_dev
],
new_service_principals=[params.logfeeder_kerberos_principal])
def enable_kms_plugin():
import params
if params.has_ranger_admin:
ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url)
ambari_username_password_for_ranger = format(
"{ambari_ranger_admin}:{ambari_ranger_password}")
response_code = ranger_adm_obj.check_ranger_login_urllib2(
params.policymgr_mgr_url)
if response_code is not None and response_code == 200:
user_resp_code = ranger_adm_obj.create_ambari_admin_user(
params.ambari_ranger_admin, params.ambari_ranger_password,
params.admin_uname_password)
else:
raise Fail('Ranger service is not started on given host')
if user_resp_code is not None and user_resp_code == 200:
get_repo_flag = get_repo(params.policymgr_mgr_url,
params.repo_name,
ambari_username_password_for_ranger)
if not get_repo_flag:
create_repo(params.policymgr_mgr_url,
json.dumps(params.kms_ranger_plugin_repo),
ambari_username_password_for_ranger)
else:
raise Fail('Ambari admin user creation failed')
current_datetime = datetime.now()
File(
format('{kms_conf_dir}/ranger-security.xml'),
owner=params.kms_user,
group=params.kms_group,
mode=0644,
content=InlineTemplate(
format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
)))
Directory([
os.path.join('/etc', 'ranger', params.repo_name),
os.path.join('/etc', 'ranger', params.repo_name, 'policycache')
],
owner=params.kms_user,
group=params.kms_group,
mode=0775,
recursive=True)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',
format('kms_{repo_name}.json')),
owner=params.kms_user,
group=params.kms_group,
mode=0644)
XmlConfig(
"ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-audit'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-security'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-policymgr-ssl'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'auditDBCred', '-v',
PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.credential_file,
owner=params.kms_user,
group=params.kms_group,
mode=0640)
def service_check(self, env):
import params
env.set_params(params)
try:
params.HdfsResource(params.revo_share_hdfs_folder,
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
mode=0777)
params.HdfsResource(None, action="execute")
except Exception as exception:
Logger.warning(
"Could not check the existence of /user/RevoShare on HDFS, exception: {0}"
.format(str(exception)))
if params.security_enabled:
kinit_cmd = format(
"{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};"
)
Execute(kinit_cmd, user=params.smokeuser)
output_file = format('{tmp_dir}/microsoft_r_server_serviceCheck.out')
File(format("{tmp_dir}/microsoft_r_server_serviceCheck.r"),
content=StaticFile("microsoft_r_server_serviceCheck.r"),
mode=0755)
Execute(format(
"Revo64 --no-save < {tmp_dir}/microsoft_r_server_serviceCheck.r | tee {output_file}"
),
tries=1,
try_sleep=1,
path=format('/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
user=params.smokeuser,
logoutput=True)
# Verify correct output
from resource_management.core import sudo
output_content = sudo.read_file(format('{output_file}'))
import re
values_list = re.findall(
r"\s(Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)\s+(\d*)",
output_content)
if 7 != len(values_list):
Logger.info("Unable to verify output of service check run")
raise Fail("Unable to verify output of service check run")
dayCountDictionary = {
'Monday': '97975',
'Tuesday': '77725',
'Wednesday': '78875',
'Thursday': '81304',
'Friday': '82987',
'Saturday': '86159',
'Sunday': '94975'
}
for (day, count) in values_list:
if count != dayCountDictionary[day]:
Logger.info(
"Service check produced incorrect output for {0}. Was expecting {1} but encountered {2}"
.format(day, dayCountDictionary[day], count))
raise Fail(
"Service check produced incorrect output for {0}. Was expecting {1} but encountered {2}"
.format(day, dayCountDictionary[day], count))
def configure(self, env):
import params
import status_params
env.set_params(params)
env.set_params(status_params)
self.create_profile_types_atlas()
self.create_ranger_policy(env)
self.create_dpprofiler_log_dir(env)
self.create_dpprofiler_dir(params)
Logger.info("Creating pid directory")
Directory([params.dpprofiler_pid_dir],
owner=params.dpprofiler_user,
group=params.dpprofiler_group,
cd_access="a",
create_parents=True,
mode=0755
)
Directory([params.dpprofiler_conf_dir],
owner='root',
group='root',
cd_access="a",
create_parents=True,
mode=0755
)
Logger.info("Creating symlink to Profiler Agent conf directory")
Execute(format('{sudo} ln -s {params.dpprofiler_conf_dir} {params.dpprofiler_home}'),
ignore_failures=True)
Logger.info("Writing conf files")
File(os.path.join(params.dpprofiler_conf_dir, 'application.conf'),
owner=params.dpprofiler_user,
group=params.dpprofiler_group,
mode=0600,
content=Template("application.conf.j2")
)
File(os.path.join(params.dpprofiler_conf_dir, 'flyway.conf'),
owner=params.dpprofiler_user,
group=params.dpprofiler_group,
mode=0600,
content=Template("flyway.conf.j2")
)
File(os.path.join(params.dpprofiler_conf_dir, 'clusterconfigs.conf'),
owner=params.dpprofiler_user,
group=params.dpprofiler_group,
mode=0600,
content=Template("clusterconfigs.conf.j2")
)
File(os.path.join(params.dpprofiler_conf_dir, 'livyconfigs.conf'),
owner=params.dpprofiler_user,
group=params.dpprofiler_group,
mode=0600,
content=Template("livyconfigs.conf.j2")
)
File(os.path.join(params.dpprofiler_conf_dir, 'dpprofiler_job_configs.conf'),
owner=params.dpprofiler_user,
group=params.dpprofiler_group,
mode=0600,
content=Template("dpprofiler_job_configs.conf.j2")
)
if params.dpprofiler_secured:
File(os.path.join(params.dpprofiler_conf_dir, 'krb5JAASLogin.conf'),
owner=params.dpprofiler_user,
group=params.dpprofiler_group,
mode=0600,
content=Template("krb5JAASLogin.conf.j2")
)
# write out logback.xml
logback_content = InlineTemplate(params.logback_content)
File(format("{params.dpprofiler_conf_dir}/logback.xml"), content=logback_content,
owner=params.dpprofiler_user, group=params.dpprofiler_group)
def configure(self, env):
import params
import status_params
env.set_params(params)
env.set_params(status_params)
self.create_zeppelin_log_dir(env)
# create the pid and zeppelin dirs
Directory([params.zeppelin_pid_dir, params.zeppelin_dir],
owner=params.zeppelin_user,
group=params.zeppelin_group,
cd_access="a",
create_parents=True,
mode=0755)
self.chown_zeppelin_pid_dir(env)
# write out zeppelin-site.xml
XmlConfig(
"zeppelin-site.xml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['zeppelin-config'],
owner=params.zeppelin_user,
group=params.zeppelin_group)
# write out zeppelin-env.sh
env_content = InlineTemplate(params.zeppelin_env_content)
File(format("{params.conf_dir}/zeppelin-env.sh"),
content=env_content,
owner=params.zeppelin_user,
group=params.zeppelin_group)
# write out shiro.ini
shiro_ini_content = InlineTemplate(params.shiro_ini_content)
File(format("{params.conf_dir}/shiro.ini"),
content=shiro_ini_content,
owner=params.zeppelin_user,
group=params.zeppelin_group)
# write out log4j.properties
File(format("{params.conf_dir}/log4j.properties"),
content=params.log4j_properties_content,
owner=params.zeppelin_user,
group=params.zeppelin_group)
self.create_zeppelin_hdfs_conf_dir(env)
if len(params.hbase_master_hosts) > 0 and params.is_hbase_installed:
# copy hbase-site.xml
XmlConfig(
"hbase-site.xml",
conf_dir=params.external_dependency_conf,
configurations=params.config['configurations']['hbase-site'],
configuration_attributes=params.
config['configuration_attributes']['hbase-site'],
owner=params.zeppelin_user,
group=params.zeppelin_group,
mode=0644)
XmlConfig(
"hdfs-site.xml",
conf_dir=params.external_dependency_conf,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.
config['configuration_attributes']['hdfs-site'],
owner=params.zeppelin_user,
group=params.zeppelin_group,
mode=0644)
XmlConfig(
"core-site.xml",
conf_dir=params.external_dependency_conf,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.
config['configuration_attributes']['core-site'],
owner=params.zeppelin_user,
group=params.zeppelin_group,
mode=0644)
def upgrade_schema(self, env):
"""
Executes the schema upgrade binary. This is its own function because it could
be called as a standalone task from the upgrade pack, but is safe to run it for each
metastore instance. The schema upgrade on an already upgraded metastore is a NOOP.
The metastore schema upgrade requires a database driver library for most
databases. During an upgrade, it's possible that the library is not present,
so this will also attempt to copy/download the appropriate driver.
This function will also ensure that configurations are written out to disk before running
since the new configs will most likely not yet exist on an upgrade.
Should not be invoked for a DOWNGRADE; Metastore only supports schema upgrades.
"""
Logger.info("Upgrading Hive Metastore Schema")
import status_params
import params
env.set_params(params)
# ensure that configurations are written out before trying to upgrade the schema
# since the schematool needs configs and doesn't know how to use the hive conf override
self.configure(env)
if params.security_enabled:
cached_kinit_executor(status_params.kinit_path_local,
status_params.hive_user,
params.hive_metastore_keytab_path,
params.hive_metastore_principal,
status_params.hostname,
status_params.tmp_dir)
# ensure that the JDBC drive is present for the schema tool; if it's not
# present, then download it first
if params.hive_jdbc_driver in params.hive_jdbc_drivers_list:
target_directory = format("{stack_root}/{version}/hive/lib")
# download it if it does not exist
if not os.path.exists(params.source_jdbc_file):
jdbc_connector(params.hive_jdbc_target, params.hive_previous_jdbc_jar)
target_directory_and_filename = os.path.join(target_directory, os.path.basename(params.source_jdbc_file))
if params.sqla_db_used:
target_native_libs_directory = format("{target_directory}/native/lib64")
Execute(format("yes | {sudo} cp {jars_in_hive_lib} {target_directory}"))
Directory(target_native_libs_directory, create_parents = True)
Execute(format("yes | {sudo} cp {libs_in_hive_lib} {target_native_libs_directory}"))
Execute(format("{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*"))
else:
# copy the JDBC driver from the older metastore location to the new location only
# if it does not already exist
if not os.path.exists(target_directory_and_filename):
Execute(('cp', params.source_jdbc_file, target_directory),
path=["/bin", "/usr/bin/"], sudo = True)
File(target_directory_and_filename, mode = 0644)
# build the schema tool command
binary = format("{hive_schematool_ver_bin}/schematool")
# the conf.server directory changed locations between stack versions
# since the configurations have not been written out yet during an upgrade
# we need to choose the original legacy location
schematool_hive_server_conf_dir = params.hive_server_conf_dir
if params.current_version is not None:
current_version = format_stack_version(params.current_version)
if not(check_stack_feature(StackFeature.CONFIG_VERSIONING, current_version)):
schematool_hive_server_conf_dir = LEGACY_HIVE_SERVER_CONF
env_dict = {
'HIVE_CONF_DIR': schematool_hive_server_conf_dir
}
command = format("{binary} -dbType {hive_metastore_db_type} -upgradeSchema")
Execute(command, user=params.hive_user, tries=1, environment=env_dict, logoutput=True)
def set_geo_configured(self):
Logger.info("Setting GEO Configured to True")
File(self.__params.enrichment_geo_configured_flag_file,
content="",
owner=self.__params.metron_user,
mode=0755)
def druid(upgrade_type=None, nodeType=None):
import params
ensure_base_directories()
# Environment Variables
File(format("{params.druid_conf_dir}/druid-env.sh"),
owner=params.druid_user,
content=InlineTemplate(params.druid_env_sh_template))
# common config
druid_common_config = mutable_config_dict(
params.config['configurations']['druid-common'])
# User cannot override below configs
druid_common_config['druid.host'] = params.hostname
druid_common_config[
'druid.extensions.directory'] = params.druid_extensions_dir
druid_common_config[
'druid.extensions.hadoopDependenciesDir'] = params.druid_hadoop_dependencies_dir
druid_common_config[
'druid.selectors.indexing.serviceName'] = params.config[
'configurations']['druid-overlord']['druid.service']
druid_common_config['druid.selectors.coordinator.serviceName'] = \
params.config['configurations']['druid-coordinator']['druid.service']
# delete the password and user if empty otherwiswe derby will fail.
if 'derby' == druid_common_config['druid.metadata.storage.type']:
del druid_common_config['druid.metadata.storage.connector.user']
del druid_common_config['druid.metadata.storage.connector.password']
druid_env_config = mutable_config_dict(
params.config['configurations']['druid-env'])
PropertiesFile(
"common.runtime.properties",
dir=params.druid_common_conf_dir,
properties=druid_common_config,
owner=params.druid_user,
group=params.user_group,
)
Logger.info("Created common.runtime.properties")
File(format("{params.druid_common_conf_dir}/druid-log4j.xml"),
mode=0644,
owner=params.druid_user,
group=params.user_group,
content=InlineTemplate(params.log4j_props))
Logger.info("Created log4j file")
File("/etc/logrotate.d/druid",
mode=0644,
owner='root',
group='root',
content=InlineTemplate(params.logrotate_props))
Logger.info("Created log rotate file")
# Write Hadoop Configs if configured
if 'core-site' in params.config['configurations']:
XmlConfig(
"core-site.xml",
conf_dir=params.druid_common_conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configuration_attributes']
['core-site'],
owner=params.druid_user,
group=params.user_group)
if 'mapred-site' in params.config['configurations']:
XmlConfig(
"mapred-site.xml",
conf_dir=params.druid_common_conf_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configuration_attributes']
['mapred-site'],
owner=params.druid_user,
group=params.user_group)
if 'yarn-site' in params.config['configurations']:
XmlConfig(
"yarn-site.xml",
conf_dir=params.druid_common_conf_dir,
configurations=params.config['configurations']['yarn-site'],
configuration_attributes=params.config['configuration_attributes']
['yarn-site'],
owner=params.druid_user,
group=params.user_group)
if 'hdfs-site' in params.config['configurations']:
XmlConfig(
"hdfs-site.xml",
conf_dir=params.druid_common_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configuration_attributes']
['hdfs-site'],
owner=params.druid_user,
group=params.user_group)
# node specific configs
for node_type in [
'coordinator', 'overlord', 'historical', 'broker', 'middleManager',
'router'
]:
node_config_dir = format('{params.druid_conf_dir}/{node_type}')
node_type_lowercase = node_type.lower()
# Write runtime.properties file
node_config = mutable_config_dict(params.config['configurations'][
format('druid-{node_type_lowercase}')])
PropertiesFile(
"runtime.properties",
dir=node_config_dir,
properties=node_config,
owner=params.druid_user,
group=params.user_group,
)
Logger.info(
format("Created druid-{node_type_lowercase} runtime.properties"))
# Write jvm configs
File(
format('{node_config_dir}/jvm.config'),
owner=params.druid_user,
group=params.user_group,
content=InlineTemplate(
"-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
node_heap_memory=druid_env_config[format(
'druid.{node_type_lowercase}.jvm.heap.memory')],
log4j_config_file=format(
"{params.druid_common_conf_dir}/druid-log4j.xml"),
node_direct_memory=druid_env_config[format(
'druid.{node_type_lowercase}.jvm.direct.memory')],
node_jvm_opts=druid_env_config[format(
'druid.{node_type_lowercase}.jvm.opts')]))
Logger.info(format("Created druid-{node_type_lowercase} jvm.config"))
# All druid nodes have dependency on hdfs_client
ensure_hadoop_directories()
download_database_connector_if_needed()
# Pull all required dependencies
pulldeps()
def kms(upgrade_type=None):
import params
if params.has_ranger_admin:
Directory(params.kms_conf_dir,
owner = params.kms_user,
group = params.kms_group,
create_parents = True
)
Directory("/etc/security/serverKeys",
create_parents = True,
cd_access = "a"
)
Directory("/etc/ranger/kms",
create_parents = True,
cd_access = "a"
)
copy_jdbc_connector()
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
mode = 0644,
)
cp = format("{check_db_connection_jar}")
if params.db_flavor.lower() == 'sqla':
cp = cp + os.pathsep + format("{kms_home}/ews/webapp/lib/sajdbc4.jar")
else:
path_to_jdbc = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")
if not os.path.isfile(path_to_jdbc):
path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + \
params.default_connectors_map[params.db_flavor.lower()] if params.db_flavor.lower() in params.default_connectors_map else None
if not os.path.isfile(path_to_jdbc):
path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + "*"
error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.db_flavor] + \
" in ranger kms lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'"
Logger.error(error_message)
cp = cp + os.pathsep + path_to_jdbc
db_connection_check_command = format(
"{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_kms_jdbc_connection_url}' {db_user} {db_password!p} {ranger_kms_jdbc_driver}")
env_dict = {}
if params.db_flavor.lower() == 'sqla':
env_dict = {'LD_LIBRARY_PATH':params.ld_library_path}
Execute(db_connection_check_command, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10, environment=env_dict)
if params.xa_audit_db_is_enabled and params.driver_source is not None and not params.driver_source.endswith("/None"):
if params.xa_previous_jdbc_jar and os.path.isfile(params.xa_previous_jdbc_jar):
File(params.xa_previous_jdbc_jar, action='delete')
File(params.downloaded_connector_path,
content = DownloadSource(params.driver_source),
mode = 0644
)
Execute(('cp', '--remove-destination', params.downloaded_connector_path, params.driver_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_target, mode=0644)
Directory(os.path.join(params.kms_home, 'ews', 'webapp', 'WEB-INF', 'classes', 'lib'),
mode=0755,
owner=params.kms_user,
group=params.kms_group
)
Execute(('cp',format('{kms_home}/ranger-kms-initd'),'/etc/init.d/ranger-kms'),
not_if=format('ls /etc/init.d/ranger-kms'),
only_if=format('ls {kms_home}/ranger-kms-initd'),
sudo=True)
File('/etc/init.d/ranger-kms',
mode = 0755
)
Directory(format('{kms_home}/'),
owner = params.kms_user,
group = params.kms_group,
recursive_ownership = True,
)
Directory(params.ranger_kms_pid_dir,
mode=0755,
owner = params.kms_user,
group = params.user_group,
cd_access = "a",
create_parents=True
)
if params.stack_supports_pid:
File(format('{kms_conf_dir}/ranger-kms-env-piddir.sh'),
content = format("export RANGER_KMS_PID_DIR_PATH={ranger_kms_pid_dir}\nexport KMS_USER={kms_user}"),
owner = params.kms_user,
group = params.kms_group,
mode=0755
)
Directory(params.kms_log_dir,
owner = params.kms_user,
group = params.kms_group,
cd_access = 'a',
create_parents=True,
mode=0755
)
File(format('{kms_conf_dir}/ranger-kms-env-logdir.sh'),
content = format("export RANGER_KMS_LOG_DIR={kms_log_dir}"),
owner = params.kms_user,
group = params.kms_group,
mode=0755
)
Execute(('ln','-sf', format('{kms_home}/ranger-kms'),'/usr/bin/ranger-kms'),
not_if=format('ls /usr/bin/ranger-kms'),
only_if=format('ls {kms_home}/ranger-kms'),
sudo=True)
File('/usr/bin/ranger-kms', mode = 0755)
Execute(('ln','-sf', format('{kms_home}/ranger-kms'),'/usr/bin/ranger-kms-services.sh'),
not_if=format('ls /usr/bin/ranger-kms-services.sh'),
only_if=format('ls {kms_home}/ranger-kms'),
sudo=True)
File('/usr/bin/ranger-kms-services.sh', mode = 0755)
Execute(('ln','-sf', format('{kms_home}/ranger-kms-initd'),format('{kms_home}/ranger-kms-services.sh')),
not_if=format('ls {kms_home}/ranger-kms-services.sh'),
only_if=format('ls {kms_home}/ranger-kms-initd'),
sudo=True)
File(format('{kms_home}/ranger-kms-services.sh'), mode = 0755)
Directory(params.kms_log_dir,
owner = params.kms_user,
group = params.kms_group,
mode = 0775
)
do_keystore_setup(params.credential_provider_path, params.jdbc_alias, params.db_password)
do_keystore_setup(params.credential_provider_path, params.masterkey_alias, params.kms_master_key_password)
if params.stack_support_kms_hsm and params.enable_kms_hsm:
do_keystore_setup(params.credential_provider_path, params.hms_partition_alias, unicode(params.hms_partition_passwd))
if params.stack_supports_ranger_kms_ssl and params.ranger_kms_ssl_enabled:
do_keystore_setup(params.ranger_kms_cred_ssl_path, params.ranger_kms_ssl_keystore_alias, params.ranger_kms_ssl_passwd)
# remove plain-text password from xml configs
dbks_site_copy = {}
dbks_site_copy.update(params.config['configurations']['dbks-site'])
for prop in params.dbks_site_password_properties:
if prop in dbks_site_copy:
dbks_site_copy[prop] = "_"
XmlConfig("dbks-site.xml",
conf_dir=params.kms_conf_dir,
configurations=dbks_site_copy,
configuration_attributes=params.config['configuration_attributes']['dbks-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644
)
ranger_kms_site_copy = {}
ranger_kms_site_copy.update(params.config['configurations']['ranger-kms-site'])
if params.stack_supports_ranger_kms_ssl:
# remove plain-text password from xml configs
for prop in params.ranger_kms_site_password_properties:
if prop in ranger_kms_site_copy:
ranger_kms_site_copy[prop] = "_"
XmlConfig("ranger-kms-site.xml",
conf_dir=params.kms_conf_dir,
configurations=ranger_kms_site_copy,
configuration_attributes=params.config['configuration_attributes']['ranger-kms-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644
)
XmlConfig("kms-site.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['kms-site'],
configuration_attributes=params.config['configuration_attributes']['kms-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644
)
File(os.path.join(params.kms_conf_dir, "kms-log4j.properties"),
owner=params.kms_user,
group=params.kms_group,
content=InlineTemplate(params.kms_log4j),
mode=0644
)
if params.security_enabled:
# core-site.xml linking required by setup for HDFS encryption
XmlConfig("core-site.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configuration_attributes']['core-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644
)
else:
File(format('{kms_conf_dir}/core-site.xml'), action="delete")
