def check_command(command, keyword):
cmd = "{0} | grep -E '{1}'".format(command, keyword)
Logger.info("check command: {0}".format(command))
output = Toolkit.exe(cmd)
if (output == ""):
Logger.error("command {0} not running".format(command))
raise ComponentIsNotRunning()
def check_service(service, keyword = "running|active|运行"):
cmd = "service {0} status | grep -E '{1}'".format(service, keyword)
Logger.info("check service: {0}".format(service))
output = Toolkit.exe(cmd)
if (output == ""):
Logger.error("service {0} not running".format(service))
raise ComponentIsNotRunning()
def get_hdp_version():
try:
command = 'hdp-select status hadoop-client'
return_code, hdp_output = shell.call(command, timeout=20)
except Exception, e:
Logger.error(str(e))
raise Fail('Unable to execute hdp-select command to retrieve the version.')
def pre_rolling_upgrade_shutdown(hdfs_binary):
"""
Runs the "shutdownDatanode {ipc_address} upgrade" command to shutdown the
DataNode in preparation for an upgrade. This will then periodically check
"getDatanodeInfo" to ensure the DataNode has shutdown correctly.
This function will obtain the Kerberos ticket if security is enabled.
:param hdfs_binary: name/path of the HDFS binary to use
:return: Return True if ran ok (even with errors), and False if need to stop the datanode forcefully.
"""
import params
Logger.info('DataNode executing "shutdownDatanode" command in preparation for upgrade...')
if params.security_enabled:
Execute(params.dn_kinit_cmd, user = params.hdfs_user)
dfsadmin_base_command = get_dfsadmin_base_command(hdfs_binary)
command = format('{dfsadmin_base_command} -shutdownDatanode {dfs_dn_ipc_address} upgrade')
code, output = shell.call(command, user=params.hdfs_user)
if code == 0:
# verify that the datanode is down
_check_datanode_shutdown(hdfs_binary)
else:
# Due to bug HDFS-7533, DataNode may not always shutdown during stack upgrade, and it is necessary to kill it.
if output is not None and re.search("Shutdown already in progress", output):
Logger.error("Due to a known issue in DataNode, the command {0} did not work, so will need to shutdown the datanode forcefully.".format(command))
return False
return True
def _copy_files(source_and_dest_pairs, component_user, file_owner, group_owner, kinit_if_needed):
"""
:param source_and_dest_pairs: List of tuples (x, y), where x is the source file in the local file system,
and y is the destination file path in HDFS
:param component_user: User that will execute the Hadoop commands, usually smokeuser
:param file_owner: Owner to set for the file copied to HDFS (typically hdfs account)
:param group_owner: Owning group to set for the file copied to HDFS (typically hadoop group)
:param kinit_if_needed: kinit command if it is needed, otherwise an empty string
:return: Returns 0 if at least one file was copied and no exceptions occurred, and 1 otherwise.
Must kinit before calling this function.
"""
import params
return_value = 1
if source_and_dest_pairs and len(source_and_dest_pairs) > 0:
return_value = 0
for (source, destination) in source_and_dest_pairs:
try:
destination_dir = os.path.dirname(destination)
params.HdfsDirectory(destination_dir,
action="create",
owner=file_owner,
hdfs_user=params.hdfs_user, # this will be the user to run the commands as
mode=0555
)
# Because CopyFromLocal does not guarantee synchronization, it's possible for two processes to first attempt to
# copy the file to a temporary location, then process 2 fails because the temporary file was already created by
# process 1, so process 2 tries to clean up by deleting the temporary file, and then process 1
# cannot finish the copy to the final destination, and both fail!
# For this reason, the file name on the destination must be unique, and we then rename it to the intended value.
# The rename operation is synchronized by the Namenode.
orig_dest_file_name = os.path.split(destination)[1]
unique_string = str(uuid.uuid4())[:8]
new_dest_file_name = orig_dest_file_name + "." + unique_string
new_destination = os.path.join(destination_dir, new_dest_file_name)
CopyFromLocal(source,
mode=0444,
owner=file_owner,
group=group_owner,
user=params.hdfs_user, # this will be the user to run the commands as
dest_dir=destination_dir,
dest_file=new_dest_file_name,
kinnit_if_needed=kinit_if_needed,
hdfs_user=params.hdfs_user,
hadoop_bin_dir=params.hadoop_bin_dir,
hadoop_conf_dir=params.hadoop_conf_dir
)
mv_command = format("fs -mv {new_destination} {destination}")
ExecuteHadoop(mv_command,
user=params.hdfs_user,
bin_dir=params.hadoop_bin_dir,
conf_dir=params.hadoop_conf_dir
)
except Exception, e:
Logger.error("Failed to copy file. Source: %s, Destination: %s. Error: %s" % (source, destination, e.message))
return_value = 1
def check_service_status(self, service, keyword):
cmd = "service {0} status | grep -E '{1}'".format(service, keyword)
Logger.info("run service check on {0} : ".format(service))
(status, output) = commands.getstatusoutput(cmd)
if (output == ""):
Logger.error("service {0} not running".format(service))
raise ComponentIsNotRunning()
def syncCluster(self, coors, datanodes, coord_port, datanode_port, host, sql_str):
for coor in coors:
name = "coordinator_" + coor.replace(".", "_")
sql = "CREATE NODE {0} WITH (TYPE = '{1}', HOST = '{2}', PORT = {3});".format(name, "coordinator", coor, coord_port)
cmd = sql_str.format(host, coord_port, sql)
Logger.info(cmd)
(ret, out) = commands.getstatusoutput(cmd)
if (ret == 0 and out.rfind("CREATE NODE") != -1):
# insert record
Logger.info("coordinator {0} create success on coordinator {1}".format(name, host))
elif (ret != 0 and out.rfind(name) != -1):
# update record
Logger.info("coordinator {0} exist on coordinator {1}, updating coordinator ...".format(name, host))
update_cmd = cmd.replace("CREATE", "ALTER")
Logger.info(update_cmd)
(ret, out) = commands.getstatusoutput(update_cmd)
if (ret == 0 and out.rfind("ALTER NODE") != -1):
Logger.info("coordinator {0} update success on coordinator {1}".format(name, host))
elif (ret != 0 and out.rfind(name) != -1):
# update coordinator self
# update pgxc_node set node_host = 'local', node_port = 5000 WHERE node_name = 'datanode_10_151_0_123';
self_sql = "update pgxc_node set node_host = '{0}', node_port = {1} WHERE node_name = '{2}';".format(host, datanode_port, name)
self_cmd = sql_str.format(coor, coord_port, self_sql)
Logger.info(self_cmd)
(ret, out) = commands.getstatusoutput(self_cmd)
if (ret == 0 and out.rfind("UPDATE 1") != -1):
Logger.info("coordinator {0} update success on coordinator {1} itself".format(name, host))
else:
Logger.error("coordinator {0} update failed on coordinator {1} itself".format(name, host))
raise Fail()
else:
Logger.error("coordinator {0} update failed on coordinator {1}".format(name, host))
raise Fail()
else:
Logger.error("coordinator {0} create failed on coordinator {1}".format(name, host))
raise Fail()
for datanode in datanodes:
name = "datanode_" + datanode.replace(".", "_")
sql = "CREATE NODE {0} WITH (TYPE = '{1}', HOST = '{2}', PORT = {3});".format(name, "datanode", datanode, datanode_port)
cmd = sql_str.format(host, coord_port, sql)
Logger.info(cmd)
(ret, out) = commands.getstatusoutput(cmd)
if (ret == 0 and out.rfind("CREATE NODE") != -1):
# insert record
Logger.info("datanode {0} create success on coordinator {1}".format(name, host))
elif (ret != 0 and out.rfind(name) != -1):
# update record
Logger.info("datanode {0} exist on coordinator {1}, updating datanode ...".format(name, host))
update_cmd = cmd.replace("CREATE", "ALTER")
Logger.info(update_cmd)
(ret, out) = commands.getstatusoutput(update_cmd)
if (ret == 0 and out.rfind("ALTER NODE") != -1):
Logger.info("datannode {0} update success on coordinator {1}".format(name, host))
else:
Logger.error("datanode {0} update failed on coordinator {1}".format(name, host))
raise Fail()
else:
Logger.error("datanode {0} create failed on coordinator {1}".format(name, host))
raise Fail()
def check_process(keyword):
Logger.info("check process by: {0}".format(keyword))
cmd = "ps aux | grep -E '" + keyword + "' | grep -v grep | cat"
result = Toolkit.exe(cmd)
if (result == ""):
Logger.error("process checked by {0} not exist".format(keyword))
raise ComponentIsNotRunning()
def service_check(cmd, user, label):
"""
Executes a service check command that adheres to LSB-compliant
return codes. The return codes are interpreted as defined
by the LSB.
See http://refspecs.linuxbase.org/LSB_3.0.0/LSB-PDA/LSB-PDA/iniscrptact.html
for more information.
:param cmd: The service check command to execute.
:param label: The name of the service.
"""
Logger.info("Performing service check; cmd={0}, user={1}, label={2}".format(cmd, user, label))
rc, out, err = get_user_call_output(cmd, user, is_checked_call=False)
if len(err) > 0:
Logger.error(err)
if rc in [1, 2, 3]:
# if return code in [1, 2, 3], then 'program is not running' or 'program is dead'
Logger.info("{0} is not running".format(label))
raise ComponentIsNotRunning()
elif rc == 0:
# if return code = 0, then 'program is running or service is OK'
Logger.info("{0} is running".format(label))
else:
# else service state is unknown
err_msg = "{0} service check failed; cmd '{1}' returned {2}".format(label, cmd, rc)
Logger.error(err_msg)
raise ExecutionFailed(err_msg, rc, out, err)
def _get_directory_mappings_during_upgrade():
"""
Gets a dictionary of directory to archive name that represents the
directories that need to be backed up and their output tarball archive targets
:return: the dictionary of directory to tarball mappings
"""
import params
# Must be performing an Upgrade
if params.upgrade_direction is None or params.upgrade_direction != Direction.UPGRADE or \
params.upgrade_from_version is None or params.upgrade_from_version == "":
Logger.error("Function _get_directory_mappings_during_upgrade() can only be called during a Stack Upgrade in direction UPGRADE.")
return {}
# By default, use this for all stacks.
knox_data_dir = '/var/lib/knox/data'
if params.stack_name and params.stack_name.upper() == "HDP" and \
compare_versions(format_hdp_stack_version(params.upgrade_from_version), "2.3.0.0") > 0:
# Use the version that is being upgraded from.
knox_data_dir = format('/usr/hdp/{upgrade_from_version}/knox/data')
# the trailing "/" is important here so as to not include the "conf" folder itself
directories = {knox_data_dir: BACKUP_DATA_ARCHIVE, params.knox_conf_dir + "/": BACKUP_CONF_ARCHIVE}
Logger.info(format("Knox directories to backup:\n{directories}"))
return directories
def check_process(self, keyword):
Logger.info("check process with: {0}".format(keyword))
cmd = "ps aux | grep -E '" + keyword + "' | grep -v grep | cat"
result = self.exe(cmd)
if (result == ""):
Logger.error("process {0} not exist".format(keyword))
raise ComponentIsNotRunning()
def write_actual_version_to_history_file(repository_version, actual_version):
"""
Save the tuple of repository_version,actual_version to the repo version history file if the repository_version
doesn't already exist
:param repository_version: normalized repo version (without build number) as received from the server
:param actual_version: Repo version with the build number, as determined using hdp-select
:returns Return True if appended the values to the file, otherwise, return False.
"""
wrote_value = False
if repository_version is None or actual_version is None:
return
if repository_version == "" or actual_version == "":
return
value = repository_version + "," + actual_version
key_exists = False
try:
if os.path.isfile(REPO_VERSION_HISTORY_FILE):
with open(REPO_VERSION_HISTORY_FILE, "r") as f:
for line in f.readlines():
line_parts = line.split(",")
if line_parts and len(line_parts) == 2 and line_parts[0] == repository_version and line_parts[1] == actual_version:
key_exists = True
break
if not key_exists:
with open(REPO_VERSION_HISTORY_FILE, "a") as f:
f.write(repository_version + "," + actual_version + "\n")
wrote_value = True
if wrote_value:
Logger.info("Appended value \"{0}\" to file {1} to track this as a new version.".format(value, REPO_VERSION_HISTORY_FILE))
except Exception, err:
Logger.error("Failed to write to file {0} the value: {1}. Error: {2}".format(REPO_VERSION_HISTORY_FILE, value, str(err)))
def __update_sysctl_file_suse():
"""
Updates /etc/sysctl.conf file with the HAWQ parameters on SUSE.
"""
# Backup file
backup_file_name = hawq_constants.sysctl_backup_file.format(str(int(time.time())))
try:
# Generate file with kernel parameters needed by hawq to temp file
File(hawq_constants.hawq_sysctl_tmp_file, content=__convert_sysctl_dict_to_text(), owner=hawq_constants.hawq_user,
group=hawq_constants.hawq_group)
sysctl_file_dict = utils.read_file_to_dict(hawq_constants.sysctl_suse_file)
sysctl_file_dict_original = sysctl_file_dict.copy()
hawq_sysctl_dict = utils.read_file_to_dict(hawq_constants.hawq_sysctl_tmp_file)
# Merge common system file with hawq specific file
sysctl_file_dict.update(hawq_sysctl_dict)
if sysctl_file_dict_original != sysctl_file_dict:
# Backup file
Execute("cp {0} {1}".format(hawq_constants.sysctl_suse_file, backup_file_name), timeout=hawq_constants.default_exec_timeout)
# Write merged properties to file
utils.write_dict_to_file(sysctl_file_dict, hawq_constants.sysctl_suse_file)
# Reload kernel sysctl parameters from /etc/sysctl.conf
Execute("sysctl -e -p", timeout=hawq_constants.default_exec_timeout)
except Exception as e:
Logger.error("Error occurred while updating sysctl.conf file, reverting the contents" + str(e))
Execute("cp {0} {1}".format(hawq_constants.sysctl_suse_file, hawq_constants.hawq_sysctl_tmp_file))
Execute("mv {0} {1}".format(backup_file_name, hawq_constants.sysctl_suse_file), timeout=hawq_constants.default_exec_timeout)
Logger.error("Please execute `sysctl -e -p` on the command line manually to reload the contents of file {0}".format(
hawq_constants.hawq_sysctl_tmp_file))
raise Fail("Failed to update sysctl.conf file ")
def get_hdp_version():
if not options.hdp_version:
# Ubuntu returns: "stdin: is not a tty", as subprocess output.
tmpfile = tempfile.NamedTemporaryFile()
out = None
with open(tmpfile.name, 'r+') as file:
get_hdp_version_cmd = '/usr/bin/hdp-select status %s > %s' % ('hadoop-mapreduce-historyserver', tmpfile.name)
code, stdoutdata = shell.call(get_hdp_version_cmd)
out = file.read()
pass
if code != 0 or out is None:
Logger.warning("Could not verify HDP version by calling '%s'. Return Code: %s, Output: %s." %
(get_hdp_version_cmd, str(code), str(out)))
return 1
matches = re.findall(r"([\d\.]+\-\d+)", out)
hdp_version = matches[0] if matches and len(matches) > 0 else None
if not hdp_version:
Logger.error("Could not parse HDP version from output of hdp-select: %s" % str(out))
return 1
else:
hdp_version = options.hdp_version
return hdp_version
def _get_current_hiveserver_version():
"""
Runs "hive --version" and parses the result in order
to obtain the current version of hive.
:return: the hiveserver2 version, returned by "hive --version"
"""
import params
try:
# When downgrading the source version should be the version we are downgrading from
if "downgrade" == params.upgrade_direction:
if not params.downgrade_from_version:
raise Fail('The version from which we are downgrading from should be provided in \'downgrade_from_version\'')
source_version = params.downgrade_from_version
else:
source_version = params.current_version
hive_execute_path = _get_hive_execute_path(source_version)
version_hive_bin = params.hive_bin
formatted_source_version = format_hdp_stack_version(source_version)
if formatted_source_version and compare_versions(formatted_source_version, "2.2") >= 0:
version_hive_bin = format('/usr/hdp/{source_version}/hive/bin')
command = format('{version_hive_bin}/hive --version')
return_code, hdp_output = shell.call(command, user=params.hive_user, path=hive_execute_path)
except Exception, e:
Logger.error(str(e))
raise Fail('Unable to execute hive --version command to retrieve the hiveserver2 version.')
def get_write_lock_files_solr_cloud(hadoop_prefix, collections):
import params
write_locks_to_delete = ''
for collection_path in collections:
code, output = call(format('{hadoop_prefix} -ls {collection_path}'))
core_paths = get_core_paths(output, collection_path)
collection_name = collection_path.replace(format('{solr_hdfs_directory}/'), '')
zk_code, zk_output = call(format(
'{zk_client_prefix} -cmd get {solr_cloud_zk_directory}/collections/{collection_name}/state.json'),
env={'JAVA_HOME': params.java64_home},
timeout=60
)
if zk_code != 0:
Logger.error(format('Cannot determine cores owned by [{solr_hostname}] in collection [{collection_name}] due to ZK error.'))
continue
for core_path in core_paths:
core_node_name = core_path.replace(format('{collection_path}/'), '')
pattern = re.compile(format(HOSTNAME_VERIFIER_PATTERN), re.MULTILINE|re.DOTALL)
core_on_hostname = re.search(pattern, zk_output)
if core_on_hostname is not None:
write_locks_to_delete += WRITE_LOCK_PATTERN.format(core_path)
return write_locks_to_delete
def bootstrap_standby_namenode(params, use_path=False):
bin_path = os.path.join(params.hadoop_bin_dir, '') if use_path else ""
try:
iterations = 50
bootstrap_cmd = format("{bin_path}hdfs namenode -bootstrapStandby -nonInteractive")
# Blue print based deployments start both NN in parallel and occasionally
# the first attempt to bootstrap may fail. Depending on how it fails the
# second attempt may not succeed (e.g. it may find the folder and decide that
# bootstrap succeeded). The solution is to call with -force option but only
# during initial start
if params.command_phase == "INITIAL_START":
bootstrap_cmd = format("{bin_path}hdfs namenode -bootstrapStandby -nonInteractive -force")
Logger.info("Boostrapping standby namenode: %s" % (bootstrap_cmd))
for i in range(iterations):
Logger.info('Try %d out of %d' % (i+1, iterations))
code, out = shell.call(bootstrap_cmd, logoutput=False, user=params.hdfs_user)
if code == 0:
Logger.info("Standby namenode bootstrapped successfully")
return True
elif code == 5:
Logger.info("Standby namenode already bootstrapped")
return True
else:
Logger.warning('Bootstrap standby namenode failed with %d error code. Will retry' % (code))
except Exception as ex:
Logger.error('Bootstrap standby namenode threw an exception. Reason %s' %(str(ex)))
return False
def prepare_upgrade_check_for_previous_dir():
"""
During a NonRolling (aka Express Upgrade), preparing the NameNode requires backing up some data.
Check that there is no "previous" folder inside the NameNode Name Dir.
"""
import params
if params.dfs_ha_enabled:
namenode_ha = NamenodeHAState()
if namenode_ha.is_active(params.hostname):
Logger.info("NameNode High Availability is enabled and this is the Active NameNode.")
problematic_previous_namenode_dirs = set()
nn_name_dirs = params.dfs_name_dir.split(',')
for nn_dir in nn_name_dirs:
if os.path.isdir(nn_dir):
# Check for a previous folder, which is not allowed.
previous_dir = os.path.join(nn_dir, "previous")
if os.path.isdir(previous_dir):
problematic_previous_namenode_dirs.add(previous_dir)
if len(problematic_previous_namenode_dirs) > 0:
message = 'WARNING. The following NameNode Name Dir(s) have a "previous" folder from an older version.\n' \
'Please back it up first, and then delete it, OR Finalize (E.g., "hdfs dfsadmin -finalizeUpgrade").\n' \
'NameNode Name Dir(s): {0}\n' \
'***** Then, retry this step. *****'.format(", ".join(problematic_previous_namenode_dirs))
Logger.error(message)
raise Fail(message)
def check_postgre_running(self, statusCmd):
Logger.info("check service by: {0}".format(statusCmd))
cmd = "{0} | grep -E 'is running'".format(statusCmd)
result = self.exe1(cmd)
if (result != ""):
Logger.error("service not exist")
raise ComponentIsNotRunning()
def wait_for_safemode_off(hdfs_binary, afterwait_sleep=0, execute_kinit=False):
"""
During NonRolling (aka Express Upgrade), after starting NameNode, which is still in safemode, and then starting
all of the DataNodes, we need for NameNode to receive all of the block reports and leave safemode.
If HA is present, then this command will run individually on each NameNode, which checks for its own address.
"""
import params
Logger.info("Wait to leafe safemode since must transition from ON to OFF.")
if params.security_enabled and execute_kinit:
kinit_command = format("{params.kinit_path_local} -kt {params.hdfs_user_keytab} {params.hdfs_principal_name}")
Execute(kinit_command, user=params.hdfs_user, logoutput=True)
try:
# Note, this fails if namenode_address isn't prefixed with "params."
dfsadmin_base_command = get_dfsadmin_base_command(hdfs_binary, use_specific_namenode=True)
is_namenode_safe_mode_off = dfsadmin_base_command + " -safemode get | grep 'Safe mode is OFF'"
# Wait up to 30 mins
Execute(is_namenode_safe_mode_off,
tries=115,
try_sleep=10,
user=params.hdfs_user,
logoutput=True
)
# Wait a bit more since YARN still depends on block reports coming in.
# Also saw intermittent errors with HBASE service check if it was done too soon.
time.sleep(afterwait_sleep)
except Fail:
Logger.error("NameNode is still in safemode, please be careful with commands that need safemode OFF.")
def service_check(self, env):
import params
env.set_params(params)
if not os.path.isfile(params.solr_config_pid_file):
Logger.error(format("PID file {solr_config_pid_file} does not exist"))
exit(1)
if not params.solr_collection_sample_create:
Logger.info("Create sample collection unchecked, skipping ...")
return
if exists_collection(params.solr_collection_name):
Logger.warning(format("Collection {solr_collection_name} already exists, skipping ..."))
return
if not params.solr_cloud_mode:
Execute(
format(
'{solr_config_bin_dir}/solr create_core -c {solr_collection_name}' +
' -d {solr_collection_config_dir} -p {solr_config_port} >> {solr_config_service_log_file} 2>&1'
),
environment={'JAVA_HOME': params.java64_home},
user=params.solr_config_user
)
else:
Execute(format(
'{solr_config_bin_dir}/solr create_collection -c {solr_collection_name}' +
' -d {solr_collection_config_dir} -p {solr_config_port}' +
' -s {solr_collection_shards} -rf {solr_collection_replicas}' +
' >> {solr_config_service_log_file} 2>&1'),
environment={'JAVA_HOME': params.java64_home},
user=params.solr_config_user
)
def check_url(self, url):
Logger.info("check url: {0}".format(url))
cmd = "curl -I \"" + url + "\" 2> /dev/null | awk 'NR==1{print}' | awk '{print $2}'"
result = self.exe(cmd)
if (result != "200"):
Logger.error("service not exist")
raise ComponentIsNotRunning()
def delete_storm_local_data(self, env):
"""
Deletes Storm data from local directories. This will create a marker file
with JSON data representing the upgrade stack and request/stage ID. This
will prevent multiple Storm components on the same host from removing
the local directories more than once.
:return:
"""
import params
Logger.info('Clearing Storm data from local directories...')
storm_local_directory = params.local_dir
if storm_local_directory is None:
raise Fail("The storm local directory specified by storm-site/storm.local.dir must be specified")
request_id = default("/requestId", None)
stage_id = default("/stageId", None)
stack_version = params.version
stack_name = params.stack_name
json_map = {}
json_map["requestId"] = request_id
json_map["stageId"] = stage_id
json_map["stackVersion"] = stack_version
json_map["stackName"] = stack_name
temp_directory = params.tmp_dir
upgrade_file = os.path.join(temp_directory, "storm-upgrade-{0}.json".format(stack_version))
if os.path.exists(upgrade_file):
try:
with open(upgrade_file) as file_pointer:
existing_json_map = json.load(file_pointer)
if cmp(json_map, existing_json_map) == 0:
Logger.info("The storm upgrade has already removed the local directories for {0}-{1} for request {2} and stage {3}".format(
stack_name, stack_version, request_id, stage_id))
# nothing else to do here for this as it appears to have already been
# removed by another component being upgraded
return
except:
Logger.error("The upgrade file {0} appears to be corrupt; removing...".format(upgrade_file))
File(upgrade_file, action="delete")
else:
# delete the upgrade file since it does not match
File(upgrade_file, action="delete")
# delete from local directory
Directory(storm_local_directory, action="delete", recursive=True)
# recreate storm local directory
Directory(storm_local_directory, mode=0755, owner = params.storm_user,
group = params.user_group, recursive = True)
# the file doesn't exist, so create it
with open(upgrade_file, 'w') as file_pointer:
json.dump(json_map, file_pointer, indent=2)
def check_ranger_service():
import params
ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url)
ambari_username_password_for_ranger = format("{ambari_ranger_admin}:{ambari_ranger_password}")
response_code = ranger_adm_obj.check_ranger_login_urllib2(params.policymgr_mgr_url)
if response_code is not None and response_code == 200:
user_resp_code = ranger_adm_obj.create_ambari_admin_user(params.ambari_ranger_admin, params.ambari_ranger_password, params.admin_uname_password)
if user_resp_code is not None and user_resp_code == 200:
get_repo_flag = get_repo(params.policymgr_mgr_url, params.repo_name, ambari_username_password_for_ranger)
if not get_repo_flag:
create_repo_flag = create_repo(params.policymgr_mgr_url, json.dumps(params.kms_ranger_plugin_repo), ambari_username_password_for_ranger)
if create_repo_flag:
return True
else:
return False
else:
return True
else:
Logger.error('Ambari admin user creation failed')
return False
else:
Logger.error('Ranger service is not reachable host')
return False
def create_repo(url, data, usernamepassword):
try:
base_url = url + '/service/public/v2/api/service'
base64string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '')
headers = {
'Accept': 'application/json',
"Content-Type": "application/json"
}
request = urllib2.Request(base_url, data, headers)
request.add_header("Authorization", "Basic {0}".format(base64string))
result = urllib2.urlopen(request, timeout=20)
response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(result.read()))
if response_code == 200:
Logger.info('Repository created Successfully')
return True
else:
Logger.info('Repository not created')
return False
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
Logger.error("Error creating service. Http status code - {0}. \n {1}".format(e.code, e.read()))
return False
else:
Logger.error("Error creating service. Reason - {0}.".format(e.reason))
return False
def update_ranger_policy(self, policyId, data, usernamepassword):
"""
:param policyId: policy id which needs to be updated
:param data: policy data that needs to be updated
:param usernamepassword: user credentials using which policy needs to be updated
:return Returns successful response and response code else None
"""
try:
searchRepoURL = self.urlPolicies + "/" + str(policyId)
base64string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '')
headers = {
'Accept': 'application/json',
"Content-Type": "application/json"
}
request = urllib2.Request(searchRepoURL, data, headers)
request.add_header("Authorization", "Basic {0}".format(base64string))
request.get_method = lambda: 'PUT'
result = openurl(request, timeout=20)
response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(result.read()))
if response_code == 200:
Logger.info('Policy updated Successfully')
return response_code
else:
Logger.error('Update Policy failed')
return None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
raise Fail("Error updating policy. Http status code - {0}. \n {1}".format(e.code, e.read()))
else:
raise Fail("Error updating policy. Reason - {0}.".format(e.reason))
def get_repo(url, name, usernamepassword):
try:
base_url = url + '/service/public/v2/api/service?serviceName=' + name + '&serviceType=kms&isEnabled=true'
request = urllib2.Request(base_url)
base64string = base64.encodestring(usernamepassword).replace('\n', '')
request.add_header("Content-Type", "application/json")
request.add_header("Accept", "application/json")
request.add_header("Authorization", "Basic {0}".format(base64string))
result = urllib2.urlopen(request, timeout=20)
response_code = result.getcode()
response = json.loads(result.read())
if response_code == 200 and len(response) > 0:
for repo in response:
if repo.get('name').lower() == name.lower() and repo.has_key('name'):
Logger.info('KMS repository exist')
return True
else:
Logger.info('KMS repository doesnot exist')
return False
else:
Logger.info('KMS repository doesnot exist')
return False
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
Logger.error("Error getting {0} service. Http status code - {1}. \n {2}".format(name, e.code, e.read()))
return False
else:
Logger.error("Error getting {0} service. Reason - {1}.".format(name, e.reason))
return False
def status(self, env):
import params
cmd = "curl -I \"" + params.lhotse_web_url + "\" 2> /dev/null | awk 'NR==1{print}' | awk '{print $2}'"
Logger.error("run cmd = {0}".format(cmd))
(ret, output) = commands.getstatusoutput(cmd)
if output != "200" :
Logger.error("lhotse web not exists")
raise ComponentIsNotRunning()
def check_data_correctness(self):
expected_data = "55"
Logger.info("Validating data inserted, finding sum of all the inserted entries. Expected output: {0}".format(expected_data))
sql_cmd = "select sum(col1) from {0}".format(hawq_constants.smoke_check_table_name)
_, stdout, _ = exec_psql_cmd(sql_cmd, self.active_master_host, tuples_only=False)
if expected_data != stdout.strip():
Logger.error("Incorrect data returned. Expected Data: {0} Actual Data: {1}".format(expected_data, stdout))
raise Fail("Incorrect data returned.")
def check_state(self):
import params
command = "source {0} && hawq state -d {1}".format(hawq_constants.hawq_greenplum_path_file, params.hawq_master_dir)
Logger.info("Executing hawq status check..")
(retcode, out, err) = exec_ssh_cmd(self.active_master_host, command)
if retcode:
Logger.error("hawq state command returned non-zero result: {0}. Out: {1} Error: {2}".format(retcode, out, err))
raise Fail("Unexpected result of hawq state command.")
Logger.info("Output of command:\n{0}".format(str(out) + "\n"))
def enable_kms_plugin():
import params
if params.has_ranger_admin:
ranger_flag = False
if params.stack_supports_ranger_kerberos and params.security_enabled:
if not is_empty(params.rangerkms_principal
) and params.rangerkms_principal != '':
ranger_flag = check_ranger_service_support_kerberos(
params.kms_user, params.rangerkms_keytab,
params.rangerkms_principal)
else:
ranger_flag = check_ranger_service_support_kerberos(
params.kms_user, params.spengo_keytab,
params.spnego_principal)
else:
ranger_flag = check_ranger_service()
if not ranger_flag:
Logger.error('Error in Get/Create service for Ranger Kms.')
current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
File(format('{kms_conf_dir}/ranger-security.xml'),
owner=params.kms_user,
group=params.kms_group,
mode=0644,
content=format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'))
Directory([
os.path.join('/etc', 'ranger', params.repo_name),
os.path.join('/etc', 'ranger', params.repo_name, 'policycache')
],
owner=params.kms_user,
group=params.kms_group,
mode=0775,
create_parents=True)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',
format('kms_{repo_name}.json')),
owner=params.kms_user,
group=params.kms_group,
mode=0644)
# remove plain-text password from xml configs
plugin_audit_properties_copy = {}
plugin_audit_properties_copy.update(
params.config['configurations']['ranger-kms-audit'])
if params.plugin_audit_password_property in plugin_audit_properties_copy:
plugin_audit_properties_copy[
params.plugin_audit_password_property] = "crypted"
XmlConfig(
"ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=plugin_audit_properties_copy,
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-security'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
# remove plain-text password from xml configs
ranger_kms_policymgr_ssl_copy = {}
ranger_kms_policymgr_ssl_copy.update(
params.config['configurations']['ranger-kms-policymgr-ssl'])
for prop in params.kms_plugin_password_properties:
if prop in ranger_kms_policymgr_ssl_copy:
ranger_kms_policymgr_ssl_copy[prop] = "crypted"
XmlConfig(
"ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=ranger_kms_policymgr_ssl_copy,
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'auditDBCred', '-v',
PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.credential_file,
owner=params.kms_user,
group=params.kms_group,
mode=0640)
# create ranger kms audit directory
if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/kms",
type="directory",
action="create_on_execute",
owner=params.kms_user,
group=params.kms_group,
mode=0750,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
if params.xa_audit_hdfs_is_enabled and len(params.namenode_host) > 1:
Logger.info(
'Audit to Hdfs enabled in NameNode HA environment, creating hdfs-site.xml'
)
XmlConfig(
"hdfs-site.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.
config['configuration_attributes']['hdfs-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644)
else:
File(format('{kms_conf_dir}/hdfs-site.xml'), action="delete")
def create_ams_dashboards():
"""
Create dashboards in grafana from the json files
"""
import params
server = Server(protocol=params.ams_grafana_protocol.strip(),
host=params.ams_grafana_host.strip(),
port=params.ams_grafana_port,
user=params.ams_grafana_admin_user,
password=params.ams_grafana_admin_pwd)
dashboard_files = params.get_grafana_dashboard_defs()
version = params.get_ambari_version()
Logger.info("Checking dashboards to update for Ambari version : %s" %
version)
# Friendly representation of dashboard
Dashboard = namedtuple('Dashboard', ['uri', 'id', 'title', 'tags'])
existing_dashboards = []
response = perform_grafana_get_call(GRAFANA_SEARCH_BULTIN_DASHBOARDS,
server)
if response and response.status == 200:
data = response.read()
try:
datasources = json.loads(data)
except:
Logger.error(
"Unable to parse JSON response from grafana request: %s" %
GRAFANA_SEARCH_BULTIN_DASHBOARDS)
Logger.info(data)
return
for datasource in datasources:
existing_dashboards.append(
Dashboard(uri=datasource['uri'],
id=datasource['id'],
title=datasource['title'],
tags=datasource['tags']))
pass
else:
Logger.error(
"Failed to execute search query on Grafana dashboards. "
"query = %s\n statuscode = %s\n reason = %s\n data = %s\n" %
(GRAFANA_SEARCH_BULTIN_DASHBOARDS, response.status,
response.reason, response.read()))
return
Logger.debug('Dashboard definitions found = %s' % str(dashboard_files))
if dashboard_files:
for dashboard_file in dashboard_files:
try:
with open(dashboard_file, 'r') as file:
dashboard_def = json.load(file)
except Exception, e:
Logger.error('Unable to load dashboard json file %s' %
dashboard_file)
Logger.error(str(e))
continue
if dashboard_def:
update_def = True
# Make sure static json does not have id
if "id" in dashboard_def:
dashboard_def['id'] = None
# Set correct tags
if 'tags' in dashboard_def:
dashboard_def['tags'].append('builtin')
dashboard_def['tags'].append(version)
else:
dashboard_def['tags'] = ['builtin', version]
dashboard_def['overwrite'] = True
for dashboard in existing_dashboards:
if dashboard.title == dashboard_def['title']:
if version not in dashboard.tags:
# Found existing dashboard with wrong version - update dashboard
update_def = True
else:
update_def = False # Skip update
pass
if update_def:
Logger.info(
"Updating dashboard definition for %s with tags: %s" %
(dashboard_def['title'], dashboard_def['tags']))
# Discrepancy in grafana export vs import format
dashboard_def_payload = {"dashboard": dashboard_def}
paylaod = json.dumps(dashboard_def_payload).strip()
(response,
data) = perform_grafana_post_call(GRAFANA_DASHBOARDS_URL,
paylaod, server)
if response and response.status == 200:
Logger.info("Dashboard created successfully.\n %s" %
str(data))
else:
Logger.error("Failed creating dashboard: %s" %
dashboard_def['title'])
pass
else:
Logger.info('No update needed for dashboard = %s' %
dashboard_def['title'])
pass
pass
def falcon(type, action=None, upgrade_type=None):
import params
if action == 'config':
Directory(
params.falcon_pid_dir,
owner=params.falcon_user,
create_parents=True,
mode=0755,
cd_access="a",
)
Directory(
params.falcon_log_dir,
owner=params.falcon_user,
create_parents=True,
mode=0755,
cd_access="a",
)
Directory(params.falcon_webapp_dir,
owner=params.falcon_user,
create_parents=True)
Directory(params.falcon_home,
owner=params.falcon_user,
create_parents=True)
Directory(params.etc_prefix_dir, mode=0755, create_parents=True)
Directory(params.falcon_conf_dir,
owner=params.falcon_user,
create_parents=True)
File(
params.falcon_conf_dir + '/falcon-env.sh',
content=InlineTemplate(params.falcon_env_sh_template),
owner=params.falcon_user,
group=params.user_group,
)
PropertiesFile(params.falcon_conf_dir + '/client.properties',
properties=params.falcon_client_properties,
mode=0644,
owner=params.falcon_user)
PropertiesFile(params.falcon_conf_dir + '/runtime.properties',
properties=params.falcon_runtime_properties,
mode=0644,
owner=params.falcon_user)
PropertiesFile(params.falcon_conf_dir + '/startup.properties',
properties=params.falcon_startup_properties,
mode=0644,
owner=params.falcon_user)
if params.falcon_graph_storage_directory:
Directory(params.falcon_graph_storage_directory,
owner=params.falcon_user,
group=params.user_group,
mode=0775,
create_parents=True,
cd_access="a")
if params.falcon_graph_serialize_path:
Directory(params.falcon_graph_serialize_path,
owner=params.falcon_user,
group=params.user_group,
mode=0775,
create_parents=True,
cd_access="a")
# Generate atlas-application.properties.xml file
if params.falcon_atlas_support:
# If Atlas is added later than Falcon, this package will be absent.
install_atlas_hook_packages(
params.atlas_plugin_package,
params.atlas_ubuntu_plugin_package, params.host_sys_prepped,
params.agent_stack_retry_on_unavailability,
params.agent_stack_retry_count)
atlas_hook_filepath = os.path.join(params.falcon_conf_dir,
params.atlas_hook_filename)
setup_atlas_hook(SERVICE.FALCON,
params.falcon_atlas_application_properties,
atlas_hook_filepath, params.falcon_user,
params.user_group)
# Falcon 0.10 uses FALCON_EXTRA_CLASS_PATH.
# Setup symlinks for older versions.
if params.current_version_formatted and check_stack_feature(
StackFeature.FALCON_ATLAS_SUPPORT_2_3,
params.current_version_formatted):
setup_atlas_jar_symlinks("falcon", params.falcon_webinf_lib)
if type == 'server':
if action == 'config':
if params.store_uri[0:4] == "hdfs":
params.HdfsResource(params.store_uri,
type="directory",
action="create_on_execute",
owner=params.falcon_user,
mode=0755)
elif params.store_uri[0:4] == "file":
Directory(params.store_uri[7:],
owner=params.falcon_user,
create_parents=True)
# TODO change to proper mode
params.HdfsResource(params.falcon_apps_dir,
type="directory",
action="create_on_execute",
owner=params.falcon_user,
mode=0777)
# In HDP 2.4 and earlier, the data-mirroring directory was copied to HDFS.
if params.supports_data_mirroring:
params.HdfsResource(params.dfs_data_mirroring_dir,
type="directory",
action="create_on_execute",
owner=params.falcon_user,
group=params.proxyuser_group,
recursive_chown=True,
recursive_chmod=True,
mode=0770,
source=params.local_data_mirroring_dir)
# Falcon Extensions were supported in HDP 2.5 and higher.
effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(
params.version)
supports_falcon_extensions = effective_version and check_stack_feature(
StackFeature.FALCON_EXTENSIONS, effective_version)
if supports_falcon_extensions:
params.HdfsResource(params.falcon_extensions_dest_dir,
type="directory",
action="create_on_execute",
owner=params.falcon_user,
group=params.proxyuser_group,
recursive_chown=True,
recursive_chmod=True,
mode=0755,
source=params.falcon_extensions_source_dir)
# Create the extensons HiveDR store
params.HdfsResource(os.path.join(
params.falcon_extensions_dest_dir, "mirroring"),
type="directory",
action="create_on_execute",
owner=params.falcon_user,
group=params.proxyuser_group,
mode=0770)
# At least one HDFS Dir should be created, so execute the change now.
params.HdfsResource(None, action="execute")
Directory(params.falcon_local_dir,
owner=params.falcon_user,
create_parents=True,
cd_access="a")
if params.falcon_embeddedmq_enabled == True:
Directory(os.path.abspath(
os.path.join(params.falcon_embeddedmq_data, "..")),
owner=params.falcon_user,
create_parents=True)
Directory(params.falcon_embeddedmq_data,
owner=params.falcon_user,
create_parents=True)
# although Falcon's falcon-config.sh will use 'which hadoop' to figure
# this out, in an upgraded cluster, it's possible that 'which hadoop'
# still points to older binaries; it's safer to just pass in the
# hadoop home directory to use
environment_dictionary = {"HADOOP_HOME": params.hadoop_home_dir}
pid = get_user_call_output.get_user_call_output(
format("cat {server_pid_file}"),
user=params.falcon_user,
is_checked_call=False)[1]
process_exists = format("ls {server_pid_file} && ps -p {pid}")
if action == 'start':
try:
Execute(
format('{falcon_home}/bin/falcon-config.sh server falcon'),
user=params.falcon_user,
path=params.hadoop_bin_dir,
environment=environment_dictionary,
not_if=process_exists,
)
except:
show_logs(params.falcon_log_dir, params.falcon_user)
raise
if not os.path.exists(params.target_jar_file):
try:
File(params.target_jar_file,
content=DownloadSource(params.bdb_resource_name),
mode=0755)
except:
exc_msg = traceback.format_exc()
exception_message = format(
"Caught Exception while downloading {bdb_resource_name}:\n{exc_msg}"
)
Logger.error(exception_message)
if not os.path.isfile(params.target_jar_file):
error_message = """
If you are using bdb as the Falcon graph db store, please run
ambari-server setup --jdbc-db=bdb --jdbc-driver=<path to je5.0.73.jar>
on the ambari server host. Otherwise falcon startup will fail.
Otherwise please configure Falcon to use HBase as the backend as described
in the Falcon documentation.
"""
Logger.error(error_message)
try:
Execute(
format(
'{falcon_home}/bin/falcon-start -port {falcon_port}'),
user=params.falcon_user,
path=params.hadoop_bin_dir,
environment=environment_dictionary,
not_if=process_exists,
)
except:
show_logs(params.falcon_log_dir, params.falcon_user)
raise
if action == 'stop':
try:
Execute(format('{falcon_home}/bin/falcon-stop'),
user=params.falcon_user,
path=params.hadoop_bin_dir,
environment=environment_dictionary)
except:
show_logs(params.falcon_log_dir, params.falcon_user)
raise
File(params.server_pid_file, action='delete')
def hive_interactive(name=None):
import params
# list of properties that should be excluded from the config
# this approach is a compromise against adding a dedicated config
# type for hive_server_interactive or needed config groups on a
# per component basis
exclude_list = ['hive.enforce.bucketing', 'hive.enforce.sorting']
# List of configs to be excluded from hive2 client, but present in Hive2 server.
exclude_list_for_hive2_client = ['javax.jdo.option.ConnectionPassword']
# Copy Tarballs in HDFS.
if params.stack_version_formatted_major and check_stack_feature(
StackFeature.ROLLING_UPGRADE,
params.stack_version_formatted_major):
resource_created = copy_to_hdfs(
"tez_hive2",
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
host_sys_prepped=params.host_sys_prepped)
if resource_created:
params.HdfsResource(None, action="execute")
Directory(params.hive_interactive_etc_dir_prefix, mode=0755)
Logger.info("Directories to fill with configs: %s" %
str(params.hive_conf_dirs_list))
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
'''
As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml,
we need to merge hive/hive-site.xml and hive2/hive-site.xml and store it in hive2/hive-site.xml.
'''
merged_hive_interactive_site = {}
merged_hive_interactive_site.update(
params.config['configurations']['hive-site'])
merged_hive_interactive_site.update(
params.config['configurations']['hive-interactive-site'])
for item in exclude_list:
if item in merged_hive_interactive_site.keys():
del merged_hive_interactive_site[item]
'''
Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook',
which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed.
'''
remove_atlas_hook_if_exists(merged_hive_interactive_site)
'''
As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml,
we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml.
'''
merged_tez_interactive_site = {}
if 'tez-site' in params.config['configurations']:
merged_tez_interactive_site.update(
params.config['configurations']['tez-site'])
Logger.info(
"Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'."
)
else:
Logger.error(
"Tez's 'tez-site' couldn't be retrieved from passed-in configurations."
)
merged_tez_interactive_site.update(
params.config['configurations']['tez-interactive-site'])
XmlConfig(
"tez-site.xml",
conf_dir=params.tez_interactive_config_dir,
configurations=merged_tez_interactive_site,
configuration_attributes=params.config['configuration_attributes']
['tez-interactive-site'],
owner=params.tez_interactive_user,
group=params.user_group,
mode=0664)
'''
Merge properties from hiveserver2-interactive-site into hiveserver2-site
'''
merged_hiveserver2_interactive_site = {}
if 'hiveserver2-site' in params.config['configurations']:
merged_hiveserver2_interactive_site.update(
params.config['configurations']['hiveserver2-site'])
Logger.info(
"Retrieved 'hiveserver2-site' for merging with 'hiveserver2-interactive-site'."
)
else:
Logger.error(
"'hiveserver2-site' couldn't be retrieved from passed-in configurations."
)
merged_hiveserver2_interactive_site.update(
params.config['configurations']['hiveserver2-interactive-site'])
# Create config files under /etc/hive2/conf and /etc/hive2/conf/conf.server:
# hive-site.xml
# hive-env.sh
# llap-daemon-log4j2.properties
# llap-cli-log4j2.properties
# hive-log4j2.properties
# hive-exec-log4j2.properties
# beeline-log4j2.properties
hive2_conf_dirs_list = params.hive_conf_dirs_list
hive2_client_conf_path = format(
"{stack_root}/current/{component_directory}/conf")
# Making copy of 'merged_hive_interactive_site' in 'merged_hive_interactive_site_copy', and deleting 'javax.jdo.option.ConnectionPassword'
# config from there, as Hive2 client shouldn't have that config.
merged_hive_interactive_site_copy = merged_hive_interactive_site.copy()
for item in exclude_list_for_hive2_client:
if item in merged_hive_interactive_site.keys():
del merged_hive_interactive_site_copy[item]
for conf_dir in hive2_conf_dirs_list:
if conf_dir == hive2_client_conf_path:
XmlConfig(
"hive-site.xml",
conf_dir=conf_dir,
configurations=merged_hive_interactive_site_copy,
configuration_attributes=params.
config['configuration_attributes']['hive-interactive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
else:
XmlConfig(
"hive-site.xml",
conf_dir=conf_dir,
configurations=merged_hive_interactive_site,
configuration_attributes=params.
config['configuration_attributes']['hive-interactive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
XmlConfig(
"hiveserver2-site.xml",
conf_dir=conf_dir,
configurations=merged_hiveserver2_interactive_site,
configuration_attributes=params.config['configuration_attributes']
['hiveserver2-interactive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
hive_server_interactive_conf_dir = conf_dir
File(format("{hive_server_interactive_conf_dir}/hive-env.sh"),
owner=params.hive_user,
group=params.user_group,
content=InlineTemplate(params.hive_interactive_env_sh_template))
llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"),
mode=0644,
group=params.user_group,
owner=params.hive_user,
content=params.llap_daemon_log4j)
llap_cli_log4j2_filename = 'llap-cli-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"),
mode=0644,
group=params.user_group,
owner=params.hive_user,
content=params.llap_cli_log4j2)
hive_log4j2_filename = 'hive-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"),
mode=0644,
group=params.user_group,
owner=params.hive_user,
content=params.hive_log4j2)
hive_exec_log4j2_filename = 'hive-exec-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"),
mode=0644,
group=params.user_group,
owner=params.hive_user,
content=params.hive_exec_log4j2)
beeline_log4j2_filename = 'beeline-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{beeline_log4j2_filename}"),
mode=0644,
group=params.user_group,
owner=params.hive_user,
content=params.beeline_log4j2)
File(format(
"{hive_server_interactive_conf_dir}/hadoop-metrics2-llapdaemon.properties"
),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-llapdaemon.j2"))
File(format(
"{hive_server_interactive_conf_dir}/hadoop-metrics2-llaptaskscheduler.properties"
),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-llaptaskscheduler.j2"))
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'hive.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hive.conf.j2"))
if not os.path.exists(params.target_hive_interactive):
jdbc_connector(params.target_hive_interactive,
params.hive_intaractive_previous_jdbc_jar)
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}{check_db_connection_jar_name}")),
mode=0644)
File(params.start_hiveserver2_interactive_path,
mode=0755,
content=Template(format('{start_hiveserver2_interactive_script}')))
Directory(params.hive_pid_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_log_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_interactive_var_lib,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
def create_ambari_admin_user(self, ambari_admin_username,
ambari_admin_password, usernamepassword):
"""
:param ambari_admin_username: username of user to be created
:param ambari_admin_password: user password of user to be created
:param usernamepassword: user credentials using which repository needs to be searched.
:return: Returns user credentials if user exist otherwise rerutns credentials of created user.
"""
flag_ambari_admin_present = False
match = re.match('[a-zA-Z0-9_\S]+$', ambari_admin_password)
if match is None:
raise Fail(
'Invalid password given for Ranger Admin user for Ambari')
try:
url = self.url_users + '?name=' + str(ambari_admin_username)
request = urllib2.Request(url)
base_64_string = base64.encodestring(usernamepassword).replace(
'\n', '')
request.add_header("Content-Type", "application/json")
request.add_header("Accept", "application/json")
request.add_header("Authorization",
"Basic {0}".format(base_64_string))
result = openurl(request, timeout=20)
response_code = result.getcode()
response = json.loads(result.read())
if response_code == 200 and len(response['vXUsers']) >= 0:
for vxuser in response['vXUsers']:
if vxuser['name'] == ambari_admin_username:
flag_ambari_admin_present = True
break
else:
flag_ambari_admin_present = False
if flag_ambari_admin_present:
Logger.info(ambari_admin_username +
' user already exists.')
return response_code
else:
Logger.info(
ambari_admin_username +
' user is not present, creating user using given configurations'
)
url = self.url_sec_users
admin_user = dict()
admin_user['status'] = 1
admin_user['userRoleList'] = ['ROLE_SYS_ADMIN']
admin_user['name'] = ambari_admin_username
admin_user['password'] = ambari_admin_password
admin_user['description'] = ambari_admin_username
admin_user['firstName'] = ambari_admin_username
data = json.dumps(admin_user)
base_64_string = base64.encodestring(
'{0}'.format(usernamepassword)).replace('\n', '')
headers = {
'Accept': 'application/json',
"Content-Type": "application/json"
}
request = urllib2.Request(url, data, headers)
request.add_header("Authorization",
"Basic {0}".format(base_64_string))
result = openurl(request, timeout=20)
response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(
result.read()))
if response_code == 200 and response is not None:
Logger.info('Ambari admin user creation successful.')
return response_code
else:
Logger.error('Ambari admin user creation failed.')
return None
else:
return None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
raise Fail(
"Error creating ambari admin user. Http status code - {0}. \n {1}"
.format(e.code, e.read()))
else:
raise Fail(
"Error creating ambari admin user. Reason - {0}.".format(
e.reason))
def download_database_connector_if_needed():
"""
Downloads the database connector to use when connecting to the metadata storage
"""
import params
if params.registry_storage_type != 'mysql' and params.registry_storage_type != 'oracle':
# In any other case than oracle and mysql, e.g. postgres, just return.
return
if params.jdbc_driver_jar == None:
if "mysql" in params.registry_storage_type:
Logger.error(
"Failed to find mysql-java-connector jar. Make sure you followed the steps to register mysql driver"
)
Logger.info("Users should register the mysql java driver jar.")
Logger.info("yum install mysql-connector-java*")
Logger.info(
"sudo ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar"
)
raise Fail('Unable to establish jdbc connection to your ' +
params.registry_storage_type + ' instance.')
if "oracle" in params.registry_storage_type:
Logger.error(
"Failed to find ojdbc jar. Please download and make sure you followed the steps to register oracle jdbc driver"
)
Logger.info("Users should register the oracle ojdbc driver jar.")
Logger.info(
"Create a symlink e.g. ln -s /usr/share/java/ojdbc6.jar /usr/share/java/ojdbc.jar"
)
Logger.info(
"sudo ambari-server setup --jdbc-db=oracle --jdbc-driver=/usr/share/java/ojdbc.jar"
)
raise Fail('Unable to establish jdbc connection to your ' +
params.registry_storage_type + ' instance.')
File(params.check_db_connection_jar,
content=DownloadSource(
format("{jdk_location}/{check_db_connection_jar_name}")))
target_jar_with_directory = params.connector_download_dir + os.path.sep + params.jdbc_driver_jar
target_jar_bootstrap_dir = params.connector_bootstrap_download_dir + os.path.sep + params.jdbc_driver_jar
if not os.path.exists(target_jar_with_directory):
File(params.downloaded_custom_connector,
content=DownloadSource(params.connector_curl_source))
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
target_jar_with_directory),
path=["/bin", "/usr/bin/"],
sudo=True)
File(target_jar_with_directory, owner="root", group=params.user_group)
if not os.path.exists(target_jar_bootstrap_dir):
File(params.downloaded_custom_connector,
content=DownloadSource(params.connector_curl_source))
Execute(('cp', '--remove-destination',
params.downloaded_custom_connector, target_jar_bootstrap_dir),
path=["/bin", "/usr/bin/"],
sudo=True)
File(target_jar_with_directory, owner="root", group=params.user_group)
def copy_jdbc_connector(stack_version=None):
import params
if params.jdbc_jar_name is None and params.driver_curl_source.endswith(
"/None"):
error_message = "Error! Sorry, but we can't find jdbc driver related to {0} database to download from {1}. \
Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'".format(
params.db_flavor, params.jdk_location)
Logger.error(error_message)
if params.driver_curl_source and not params.driver_curl_source.endswith(
"/None"):
if params.previous_jdbc_jar and os.path.isfile(
params.previous_jdbc_jar):
File(params.previous_jdbc_jar, action='delete')
kms_home = params.kms_home
if stack_version is not None:
kms_home = format("{stack_root}/{stack_version}/ranger-kms")
driver_curl_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source),
mode=0644)
Directory(os.path.join(kms_home, 'ews', 'lib'), mode=0755)
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C',
params.tmp_dir),
sudo=True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive,
os.path.join(kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
Directory(params.jdbc_libs_dir, cd_access="a", create_parents=True)
Execute(as_sudo([
'yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir
],
auto_escape=False),
path=["/bin", "/usr/bin/"])
File(os.path.join(kms_home, 'ews', 'webapp', 'lib', 'sajdbc4.jar'),
mode=0644)
else:
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
os.path.join(kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(kms_home, 'ews', 'webapp', 'lib',
params.jdbc_jar_name),
mode=0644)
ModifyPropertiesFile(
format("{kms_home}/install.properties"),
properties=params.config['configurations']['kms-properties'],
owner=params.kms_user)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(
format("{kms_home}/install.properties"),
properties={
'SQL_CONNECTOR_JAR':
format('{kms_home}/ews/webapp/lib/sajdbc4.jar')
},
owner=params.kms_user,
)
else:
ModifyPropertiesFile(
format("{kms_home}/install.properties"),
properties={'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
owner=params.kms_user,
)
def get_tarball_paths(name,
use_upgrading_version_during_upgrade=True,
custom_source_file=None,
custom_dest_file=None):
"""
For a given tarball name, get the source and destination paths to use.
:param name: Tarball name
:param use_upgrading_version_during_upgrade:
:param custom_source_file: If specified, use this source path instead of the default one from the map.
:param custom_dest_file: If specified, use this destination path instead of the default one from the map.
:return: A tuple of (success status, source path, destination path, optional preparation function which is invoked to setup the tarball)
"""
stack_name = Script.get_stack_name()
if not stack_name:
Logger.error(
"Cannot copy {0} tarball to HDFS because stack name could not be determined."
.format(str(name)))
return False, None, None
if name is None or name.lower() not in TARBALL_MAP:
Logger.error(
"Cannot copy tarball to HDFS because {0} is not supported in stack {1} for this operation."
.format(str(name), str(stack_name)))
return False, None, None
service = TARBALL_MAP[name.lower()]['service']
stack_version = get_current_version(service=service,
use_upgrading_version_during_upgrade=
use_upgrading_version_during_upgrade)
if not stack_version:
Logger.error(
"Cannot copy {0} tarball to HDFS because stack version could be be determined."
.format(str(name)))
return False, None, None
stack_root = Script.get_stack_root()
if not stack_root:
Logger.error(
"Cannot copy {0} tarball to HDFS because stack root could be be determined."
.format(str(name)))
return False, None, None
(source_file, dest_file) = TARBALL_MAP[name.lower()]['dirs']
if custom_source_file is not None:
source_file = custom_source_file
if custom_dest_file is not None:
dest_file = custom_dest_file
source_file = source_file.replace(STACK_NAME_PATTERN, stack_name.lower())
dest_file = dest_file.replace(STACK_NAME_PATTERN, stack_name.lower())
source_file = source_file.replace(STACK_ROOT_PATTERN, stack_root.lower())
dest_file = dest_file.replace(STACK_ROOT_PATTERN, stack_root.lower())
source_file = source_file.replace(STACK_VERSION_PATTERN, stack_version)
dest_file = dest_file.replace(STACK_VERSION_PATTERN, stack_version)
prepare_function = None
if "prepare_function" in TARBALL_MAP[name.lower()]:
prepare_function = TARBALL_MAP[name.lower()]['prepare_function']
return True, source_file, dest_file, prepare_function
except Exception, e:
Logger.logger.exception(
"Could not parse output of {0}. Error: {1}".format(
str(tmp_file), str(e)))
finally:
try:
if os.path.exists(tmp_file):
os.remove(tmp_file)
except Exception, e:
Logger.logger.exception(
"Could not remove file {0}. Error: {1}".format(
str(tmp_file), str(e)))
if code != 0 or out is None or out == "":
Logger.error(
"Could not verify stack version by calling '{0}'. Return Code: {1}, Output: {2}."
.format(get_stack_versions_cmd, str(code), str(out)))
return None
matches = re.findall(r"([\d\.]+(?:-\d+)?)", out)
if matches and len(matches) == 1:
stack_version = matches[0]
elif matches and len(matches) > 1:
Logger.error(
"Found multiple matches for stack version, cannot identify the correct one from: {0}"
.format(", ".join(matches)))
return stack_version
if 'httpProxyPassword' in config['configurations']['streamline-common']:
http_proxy_password = config['configurations']['streamline-common'][
'httpProxyPassword']
else:
http_proxy_password = None
streamline_catalog_root_url = 'http://{0}:{1}/api/v1/catalog'.format(
hostname, streamline_port)
# mysql jar
jdk_location = config['hostLevelParams']['jdk_location']
if 'mysql' == streamline_storage_type:
jdbc_driver_jar = default("/hostLevelParams/custom_mysql_jdbc_name", None)
if jdbc_driver_jar == None:
Logger.error(
"Failed to find mysql-java-connector jar. Make sure you followed the steps to register mysql driver"
)
Logger.info("Users should register the mysql java driver jar.")
Logger.info("yum install mysql-connector-java*")
Logger.info(
"sudo ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar"
)
connector_curl_source = format("{jdk_location}/{jdbc_driver_jar}")
connector_download_dir = format("{streamline_home}/libs")
connector_bootstrap_download_dir = format(
"{streamline_home}/bootstrap/lib")
downloaded_custom_connector = format("{tmp_dir}/{jdbc_driver_jar}")
check_db_connection_jar_name = "DBConnectionVerification.jar"
check_db_connection_jar = format(
def enable_kms_plugin():
import params
if params.has_ranger_admin:
count = 0
while count < 5:
ranger_flag = check_ranger_service()
if ranger_flag:
break
else:
time.sleep(5) # delay for 5 seconds
count = count + 1
else:
Logger.error(
"Ranger service is not reachable after {0} tries".format(
count))
current_datetime = datetime.now()
File(
format('{kms_conf_dir}/ranger-security.xml'),
owner=params.kms_user,
group=params.kms_group,
mode=0644,
content=InlineTemplate(
format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
)))
Directory([
os.path.join('/etc', 'ranger', params.repo_name),
os.path.join('/etc', 'ranger', params.repo_name, 'policycache')
],
owner=params.kms_user,
group=params.kms_group,
mode=0775,
recursive=True)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',
format('kms_{repo_name}.json')),
owner=params.kms_user,
group=params.kms_group,
mode=0644)
XmlConfig(
"ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['ranger-kms-audit'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-security'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-policymgr-ssl'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'auditDBCred', '-v',
PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.credential_file,
owner=params.kms_user,
group=params.kms_group,
mode=0640)
def service_check(self, env):
import params
env.set_params(params)
if params.streamline_ssl_enabled:
streamline_api = format(
"https://{params.hostname}:{params.streamline_ssl_port}/api/v1/catalog/streams/componentbundles"
)
else:
streamline_api = format(
"http://{params.hostname}:{params.streamline_port}/api/v1/catalog/streams/componentbundles"
)
Logger.info(streamline_api)
max_retries = 3
success = False
if (params.security_enabled) and (not params.streamline_sso_enabled):
kinit_cmd = format(
"{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};"
)
return_code, out = shell.checked_call(
kinit_cmd,
path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
user=params.smokeuser,
)
for num in range(0, max_retries):
try:
Logger.info(format("Making http requests to {streamline_api}"))
if params.security_enabled:
get_app_info_cmd = "curl --negotiate -u : -ks --location-trusted --connect-timeout " + CURL_CONNECTION_TIMEOUT + " " + streamline_api
return_code, stdout, _ = get_user_call_output(
get_app_info_cmd,
user=params.smokeuser,
path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
)
try:
json_response = json.loads(stdout)
success = True
Logger.info(
format(
"Successfully made a API request to SAM. {stdout}"
))
break
except Exception as e:
Logger.error(
format(
"Response from SAM API was not a valid JSON. Response: {stdout}"
))
else:
response = urllib2.urlopen(streamline_api)
api_response = response.read()
response_code = response.getcode()
Logger.info(format("SAM response http status {response}"))
if response.getcode() != 200:
Logger.error(
format(
"Failed to fetch response for {streamline_api}"
))
show_logs(params.streamline_log_dir,
params.streamline_user)
raise
else:
success = True
Logger.info(
format(
"Successfully made a API request to SAM. {api_response}"
))
break
except (urllib2.URLError, ExecutionFailed) as e:
Logger.error(
format(
"Failed to make API request to SAM server at {streamline_api},retrying.. {num} out of {max_retries}"
))
time.sleep(num * 10) #exponential back-off
continue
if success != True:
Logger.error(
format(
"Failed to make API request to SAM server at {streamline_api} after {max_retries}"
))
raise
else:
return None
except urllib2.HTTPError, e:
Logger.error(
"Error during Ranger service authentication. Http status code - {0}. {1}"
.format(e.code, e.read()))
return None
except urllib2.URLError, e:
Logger.error(
"Error during Ranger service authentication. {0}".format(
e.reason))
return None
except Exception, e:
Logger.error(
"Error occured when connecting Ranger admin. {0} ".format(e))
return None
def check_if_not_policy_exist(self, service_name):
try:
url = self.rangerurl + "/service/public/v2/api/service/" + \
service_name + "/policy?policyName=dpprofiler-audit-read"
Logger.info("Checking ranger policy. Url : {0}".format(url))
usernamepassword = '******'.format(self.username, self.password)
base_64_string = base64.encodestring(usernamepassword).replace(
'\n', '')
request = urllib2.Request(url)
request.add_header("Content-Type", "application/json")
request.add_header("Accept", "application/json")
request.add_header("Authorization",
"Basic {0}".format(base_64_string))
def _llap_start(self, env, cleanup=False):
import params
env.set_params(params)
Logger.info("Starting LLAP")
LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir()
LLAP_APP_NAME = 'llap0'
unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')
cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}"
" --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m "
" --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}"
" --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}")
if params.security_enabled:
llap_keytab_splits = params.hive_llap_keytab_file.split("/")
Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits))
cmd += format(" --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab "
"{llap_keytab_splits[4]} --slider-principal {params.hive_llap_principal}")
# Append args.
llap_java_args = InlineTemplate(params.llap_app_java_opts).get_content()
cmd += format(" --args \" {llap_java_args}\"")
run_file_path = None
try:
Logger.info(format("Command: {cmd}"))
code, output, error = shell.checked_call(cmd, user=params.hive_user, stderr=subprocess.PIPE, logoutput=True)
if code != 0 or output is None:
raise Fail("Command failed with either non-zero return code or no output.")
# E.g., output:
# Prepared llap-slider-05Apr2016/run.sh for running LLAP on Slider
exp = r"Prepared (.*?run.sh) for running LLAP"
run_file_path = None
out_splits = output.split("\n")
for line in out_splits:
line = line.strip()
m = re.match(exp, line, re.I)
if m and len(m.groups()) == 1:
run_file_name = m.group(1)
run_file_path = os.path.join(params.hive_user_home_dir, run_file_name)
break
if not run_file_path:
raise Fail("Did not find run.sh file in output: " + str(output))
Logger.info(format("Run file path: {run_file_path}"))
Execute(run_file_path, user=params.hive_user, logoutput=True)
Logger.info("Submitted LLAP app name : {0}".format(LLAP_APP_NAME))
# We need to check the status of LLAP app to figure out it got
# launched properly and is in running state. Then go ahead with Hive Interactive Server start.
status = self.check_llap_app_status(LLAP_APP_NAME, params.num_retries_for_checking_llap_status)
if status:
Logger.info("LLAP app '{0}' deployed successfully.".format(LLAP_APP_NAME))
return True
else:
Logger.error("LLAP app '{0}' deployment unsuccessful.".format(LLAP_APP_NAME))
return False
except:
# Attempt to clean up the packaged application, or potentially rename it with a .bak
if run_file_path is not None and cleanup:
try:
parent_dir = os.path.dirname(run_file_path)
if os.path.isdir(parent_dir):
shutil.rmtree(parent_dir)
except Exception, e:
Logger.error("Could not cleanup LLAP app package. Error: " + str(e))
# throw the original exception
raise
def _llap_start(self, env, cleanup=False):
import params
env.set_params(params)
LLAP_APP_NAME = 'llap0'
if params.hive_server_interactive_ha:
"""
Check llap app state
"""
Logger.info("HSI HA is enabled. Checking if LLAP is already running ...")
status = self.check_llap_app_status(LLAP_APP_NAME, 2, params.hive_server_interactive_ha)
if status:
Logger.info("LLAP app '{0}' is already running.".format(LLAP_APP_NAME))
return True
else:
Logger.info("LLAP app '{0}' is not running. llap will be started.".format(LLAP_APP_NAME))
pass
# Call for cleaning up the earlier run(s) LLAP package folders.
self._cleanup_past_llap_package_dirs()
Logger.info("Starting LLAP")
LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir()
unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')
# Figure out the Slider Anti-affinity to be used.
# YARN does not support anti-affinity, and therefore Slider implements AA by the means of exclusion lists, i.e, it
# starts containers one by one and excludes the nodes it gets (adding a delay of ~2sec./machine). When the LLAP
# container memory size configuration is more than half of YARN node memory, AA is implicit and should be avoided.
slider_placement = 4
if long(params.llap_daemon_container_size) > (0.5 * long(params.yarn_nm_mem)):
slider_placement = 0
Logger.info("Setting slider_placement : 0, as llap_daemon_container_size : {0} > 0.5 * "
"YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem))
else:
Logger.info("Setting slider_placement: 4, as llap_daemon_container_size : {0} <= 0.5 * "
"YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem))
cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}"
" --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m"
" --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}"
" --slider-placement {slider_placement} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}"
" {params.llap_extra_slider_opts} --skiphadoopversion --skiphbasecp --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}")
if params.security_enabled:
llap_keytab_splits = params.hive_llap_keytab_file.split("/")
Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits))
cmd += format(" --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab "
"{llap_keytab_splits[4]} --slider-principal {params.hive_llap_principal}")
# Add the aux jars if they are specified. If empty, dont need to add this param.
if params.hive_aux_jars:
cmd+= format(" --auxjars {params.hive_aux_jars}")
# Append args.
llap_java_args = InlineTemplate(params.llap_app_java_opts).get_content()
cmd += format(" --args \" {llap_java_args}\"")
run_file_path = None
try:
Logger.info(format("Command: {cmd}"))
code, output, error = shell.checked_call(cmd, user=params.hive_user, stderr=subprocess.PIPE, logoutput=True)
if code != 0 or output is None:
raise Fail("Command failed with either non-zero return code or no output.")
# E.g., output:
# Prepared llap-slider-05Apr2016/run.sh for running LLAP on Slider
exp = r"Prepared (.*?run.sh) for running LLAP"
run_file_path = None
out_splits = output.split("\n")
for line in out_splits:
line = line.strip()
m = re.match(exp, line, re.I)
if m and len(m.groups()) == 1:
run_file_name = m.group(1)
run_file_path = os.path.join(params.hive_user_home_dir, run_file_name)
break
if not run_file_path:
raise Fail("Did not find run.sh file in output: " + str(output))
Logger.info(format("Run file path: {run_file_path}"))
Execute(run_file_path, user=params.hive_user, logoutput=True)
Logger.info("Submitted LLAP app name : {0}".format(LLAP_APP_NAME))
# We need to check the status of LLAP app to figure out it got
# launched properly and is in running state. Then go ahead with Hive Interactive Server start.
status = self.check_llap_app_status(LLAP_APP_NAME, params.num_retries_for_checking_llap_status)
if status:
Logger.info("LLAP app '{0}' deployed successfully.".format(LLAP_APP_NAME))
return True
else:
Logger.error("LLAP app '{0}' deployment unsuccessful.".format(LLAP_APP_NAME))
return False
except:
# Attempt to clean up the packaged application, or potentially rename it with a .bak
if run_file_path is not None and cleanup:
parent_dir = os.path.dirname(run_file_path)
Directory(parent_dir,
action = "delete",
ignore_failures = True,
)
# throw the original exception
raise
raise Exception("Code is nonzero or output is empty")
Logger.debug("Command: %s\nOutput: %s" %
(get_stack_comp_version_cmd, str(out)))
matches = re.findall(r"( [\d\.]+(\-\d+)?)", out)
version = matches[0][0].strip() if matches and len(
matches) > 0 and len(matches[0]) > 0 else None
Logger.debug("Version for component %s: %s" %
(component_name, str(version)))
except Exception, e:
Logger.error(
"Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s."
% (component_name, get_stack_comp_version_cmd, str(code),
str(out)))
else:
Logger.error("Could not find stack selector for stack: %s" %
str(stack_name))
return version
def get_component_version_with_stack_selector(stack_selector_path,
component_name):
"""
For specific cases where we deal with HDP add on services from a management pack, the version
needs to be determined by using the specific stack selector itself.
:param stack_selector_path: /usr/bin/hdf-select
Comes from the service which calls for this function.
:param component_name: Component name as a string necessary to get the version
:return: Returns a string if found, e.g., 2.2.1.0-2175, otherwise, returns None
This function can be called by custom services, hence should not be removed
"""
def kms(upgrade_type=None):
import params
if params.has_ranger_admin:
Directory(params.kms_conf_dir,
owner=params.kms_user,
group=params.kms_group,
create_parents=True)
Directory("/etc/security/serverKeys",
create_parents=True,
cd_access="a")
Directory("/etc/ranger/kms", create_parents=True, cd_access="a")
copy_jdbc_connector()
File(
format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}{check_db_connection_jar_name}")),
mode=0644,
)
cp = format("{check_db_connection_jar}")
if params.db_flavor.lower() == 'sqla':
cp = cp + os.pathsep + format(
"{kms_home}/ews/webapp/lib/sajdbc4.jar")
else:
path_to_jdbc = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")
if not os.path.isfile(path_to_jdbc):
path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + \
params.default_connectors_map[params.db_flavor.lower()] if params.db_flavor.lower() in params.default_connectors_map else None
if not os.path.isfile(path_to_jdbc):
path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + "*"
error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.db_flavor] + \
" in ranger kms lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'"
Logger.error(error_message)
cp = cp + os.pathsep + path_to_jdbc
db_connection_check_command = format(
"{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_kms_jdbc_connection_url}' {db_user} {db_password!p} {ranger_kms_jdbc_driver}"
)
env_dict = {}
if params.db_flavor.lower() == 'sqla':
env_dict = {'LD_LIBRARY_PATH': params.ld_library_path}
Execute(db_connection_check_command,
path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
tries=5,
try_sleep=10,
environment=env_dict)
if params.xa_audit_db_is_enabled and params.driver_source is not None and not params.driver_source.endswith(
"/None"):
if params.xa_previous_jdbc_jar and os.path.isfile(
params.xa_previous_jdbc_jar):
File(params.xa_previous_jdbc_jar, action='delete')
File(params.downloaded_connector_path,
content=DownloadSource(params.driver_source),
mode=0644)
Execute(('cp', '--remove-destination',
params.downloaded_connector_path, params.driver_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_target, mode=0644)
Directory(os.path.join(params.kms_home, 'ews', 'webapp', 'WEB-INF',
'classes', 'lib'),
mode=0755,
owner=params.kms_user,
group=params.kms_group)
Execute(('cp', format('{kms_home}/ranger-kms-initd'),
'/etc/init.d/ranger-kms'),
not_if=format('ls /etc/init.d/ranger-kms'),
only_if=format('ls {kms_home}/ranger-kms-initd'),
sudo=True)
File('/etc/init.d/ranger-kms', mode=0755)
Directory(
format('{kms_home}/'),
owner=params.kms_user,
group=params.kms_group,
recursive_ownership=True,
)
Directory(params.ranger_kms_pid_dir,
mode=0755,
owner=params.kms_user,
group=params.user_group,
cd_access="a",
create_parents=True)
if params.stack_supports_pid:
File(
format('{kms_conf_dir}/ranger-kms-env-piddir.sh'),
content=format(
"export RANGER_KMS_PID_DIR_PATH={ranger_kms_pid_dir}\nexport KMS_USER={kms_user}"
),
owner=params.kms_user,
group=params.kms_group,
mode=0755)
Directory(params.kms_log_dir,
owner=params.kms_user,
group=params.kms_group,
cd_access='a',
create_parents=True,
mode=0755)
File(format('{kms_conf_dir}/ranger-kms-env-logdir.sh'),
content=format("export RANGER_KMS_LOG_DIR={kms_log_dir}"),
owner=params.kms_user,
group=params.kms_group,
mode=0755)
Execute(('ln', '-sf', format('{kms_home}/ranger-kms'),
'/usr/bin/ranger-kms'),
not_if=format('ls /usr/bin/ranger-kms'),
only_if=format('ls {kms_home}/ranger-kms'),
sudo=True)
File('/usr/bin/ranger-kms', mode=0755)
Execute(('ln', '-sf', format('{kms_home}/ranger-kms'),
'/usr/bin/ranger-kms-services.sh'),
not_if=format('ls /usr/bin/ranger-kms-services.sh'),
only_if=format('ls {kms_home}/ranger-kms'),
sudo=True)
File('/usr/bin/ranger-kms-services.sh', mode=0755)
Execute(('ln', '-sf', format('{kms_home}/ranger-kms-initd'),
format('{kms_home}/ranger-kms-services.sh')),
not_if=format('ls {kms_home}/ranger-kms-services.sh'),
only_if=format('ls {kms_home}/ranger-kms-initd'),
sudo=True)
File(format('{kms_home}/ranger-kms-services.sh'), mode=0755)
Directory(params.kms_log_dir,
owner=params.kms_user,
group=params.kms_group,
mode=0775)
do_keystore_setup(params.credential_provider_path, params.jdbc_alias,
params.db_password)
do_keystore_setup(params.credential_provider_path,
params.masterkey_alias,
params.kms_master_key_password)
if params.stack_support_kms_hsm and params.enable_kms_hsm:
do_keystore_setup(params.credential_provider_path,
params.hms_partition_alias,
unicode(params.hms_partition_passwd))
if params.stack_supports_ranger_kms_ssl and params.ranger_kms_ssl_enabled:
do_keystore_setup(params.ranger_kms_cred_ssl_path,
params.ranger_kms_ssl_keystore_alias,
params.ranger_kms_ssl_passwd)
# remove plain-text password from xml configs
dbks_site_copy = {}
dbks_site_copy.update(params.config['configurations']['dbks-site'])
for prop in params.dbks_site_password_properties:
if prop in dbks_site_copy:
dbks_site_copy[prop] = "_"
XmlConfig(
"dbks-site.xml",
conf_dir=params.kms_conf_dir,
configurations=dbks_site_copy,
configuration_attributes=params.config['configuration_attributes']
['dbks-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644)
ranger_kms_site_copy = {}
ranger_kms_site_copy.update(
params.config['configurations']['ranger-kms-site'])
if params.stack_supports_ranger_kms_ssl:
# remove plain-text password from xml configs
for prop in params.ranger_kms_site_password_properties:
if prop in ranger_kms_site_copy:
ranger_kms_site_copy[prop] = "_"
XmlConfig(
"ranger-kms-site.xml",
conf_dir=params.kms_conf_dir,
configurations=ranger_kms_site_copy,
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644)
XmlConfig(
"kms-site.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['kms-site'],
configuration_attributes=params.config['configuration_attributes']
['kms-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644)
File(os.path.join(params.kms_conf_dir, "kms-log4j.properties"),
owner=params.kms_user,
group=params.kms_group,
content=InlineTemplate(params.kms_log4j),
mode=0644)
if params.security_enabled:
# core-site.xml linking required by setup for HDFS encryption
XmlConfig(
"core-site.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.
config['configuration_attributes']['core-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644)
else:
File(format('{kms_conf_dir}/core-site.xml'), action="delete")
def copy_tarballs_to_hdfs(source, dest, hdp_select_component_name,
component_user, file_owner, group_owner):
"""
:param tarball_prefix: Prefix of the tarball must be one of tez, hive, mr, pig
:param hdp_select_component_name: Component name to get the status to determine the version
:param component_user: User that will execute the Hadoop commands
:param file_owner: Owner of the files copied to HDFS (typically hdfs account)
:param group_owner: Group owner of the files copied to HDFS (typically hadoop group)
:return: Returns 0 on success, 1 if no files were copied, and in some cases may raise an exception.
In order to call this function, params.py must have all of the following,
hdp_stack_version, kinit_path_local, security_enabled, hdfs_user, hdfs_principal_name, hdfs_user_keytab,
hadoop_bin_dir, hadoop_conf_dir, and HdfsDirectory as a partial function.
"""
component_tar_source_file, component_tar_destination_folder = source, dest
if not os.path.exists(component_tar_source_file):
Logger.warning("Could not find file: %s" %
str(component_tar_source_file))
return 1
# Ubuntu returns: "stdin: is not a tty", as subprocess output.
tmpfile = tempfile.NamedTemporaryFile()
out = None
with open(tmpfile.name, 'r+') as file:
get_hdp_version_cmd = '/usr/bin/hdp-select status %s > %s' % (
hdp_select_component_name, tmpfile.name)
code, stdoutdata = shell.call(get_hdp_version_cmd)
out = file.read()
pass
if code != 0 or out is None:
Logger.warning(
"Could not verify HDP version by calling '%s'. Return Code: %s, Output: %s."
% (get_hdp_version_cmd, str(code), str(out)))
return 1
matches = re.findall(r"([\d\.]+\-\d+)", out)
hdp_version = matches[0] if matches and len(matches) > 0 else None
if not hdp_version:
Logger.error(
"Could not parse HDP version from output of hdp-select: %s" %
str(out))
return 1
file_name = os.path.basename(component_tar_source_file)
destination_file = os.path.join(component_tar_destination_folder,
file_name)
destination_file = destination_file.replace("{{ hdp_stack_version }}",
hdp_version)
kinit_if_needed = ""
if params.security_enabled:
kinit_if_needed = format(
"{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};")
if kinit_if_needed:
Execute(kinit_if_needed, user=component_user, path='/bin')
does_hdfs_file_exist_cmd = "fs -ls %s" % destination_file
does_hdfs_file_exist = False
try:
ExecuteHadoop(does_hdfs_file_exist_cmd,
user=component_user,
logoutput=True,
conf_dir=params.hadoop_conf_dir,
bin_dir=params.hadoop_bin_dir)
does_hdfs_file_exist = True
except Fail:
pass
if not does_hdfs_file_exist:
source_and_dest_pairs = [
(component_tar_source_file, destination_file),
]
return _copy_files(source_and_dest_pairs, file_owner, group_owner,
kinit_if_needed)
return 1
get_stack_versions_cmd = "/usr/bin/hdp-select versions > {0}".format(tmp_file)
try:
code, stdoutdata = shell.call(get_stack_versions_cmd, logoutput=True)
with open(tmp_file, 'r+') as file:
out = file.read()
except Exception, e:
Logger.logger.exception("Could not parse output of {0}. Error: {1}".format(str(tmp_file), str(e)))
finally:
try:
if os.path.exists(tmp_file):
os.remove(tmp_file)
except Exception, e:
Logger.logger.exception("Could not remove file {0}. Error: {1}".format(str(tmp_file), str(e)))
if code != 0 or out is None or out == "":
Logger.error("Could not verify HDP version by calling '{0}'. Return Code: {1}, Output: {2}.".format(get_stack_versions_cmd, str(code), str(out)))
return None
matches = re.findall(r"([\d\.]+\-\d+)", out)
if matches and len(matches) == 1:
stack_version = matches[0]
elif matches and len(matches) > 1:
Logger.error("Found multiple matches for HDP version, cannot identify the correct one from: {0}".format(", ".join(matches)))
return stack_version
def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=None, custom_dest_file=None, force_execute=False,
use_upgrading_version_during_uprade=True, replace_existing_files=False, host_sys_prepped=False):
"""
:param name: Tarball name, e.g., tez, hive, pig, sqoop.
def create_ranger_repository(self,
component,
repo_name,
repo_properties,
ambari_ranger_admin,
ambari_ranger_password,
admin_uname,
admin_password,
policy_user,
is_security_enabled=False,
is_stack_supports_ranger_kerberos=False,
component_user=None,
component_user_principal=None,
component_user_keytab=None):
if not is_stack_supports_ranger_kerberos or not is_security_enabled:
response_code = self.check_ranger_login_urllib2(self.base_url)
repo_data = json.dumps(repo_properties)
ambari_ranger_password = unicode(ambari_ranger_password)
admin_password = unicode(admin_password)
ambari_username_password_for_ranger = format(
'{ambari_ranger_admin}:{ambari_ranger_password}')
if response_code is not None and response_code == 200:
user_resp_code = self.create_ambari_admin_user(
ambari_ranger_admin, ambari_ranger_password,
format("{admin_uname}:{admin_password}"))
if user_resp_code is not None and user_resp_code == 200:
retryCount = 0
while retryCount <= 5:
repo = self.get_repository_by_name_urllib2(
repo_name, component, 'true',
ambari_username_password_for_ranger)
if repo is not None:
Logger.info('{0} Repository {1} exist'.format(
component.title(), repo['name']))
break
else:
response = self.create_repository_urllib2(
repo_data, ambari_username_password_for_ranger)
if response is not None:
Logger.info(
'{0} Repository created in Ranger admin'.
format(component.title()))
break
else:
if retryCount < 5:
Logger.info(
"Retry Repository Creation is being called"
)
time.sleep(30) # delay for 30 seconds
retryCount += 1
else:
Logger.error(
'{0} Repository creation failed in Ranger admin'
.format(component.title()))
break
else:
Logger.error('Ambari admin user creation failed')
elif not self.skip_if_rangeradmin_down:
Logger.error("Connection failed to Ranger Admin !")
elif is_stack_supports_ranger_kerberos and is_security_enabled:
response = self.check_ranger_login_curl(component_user,
component_user_keytab,
component_user_principal,
self.url_login, True)
if response and response[0] == 200:
retryCount = 0
repo_data = json.dumps(repo_properties)
while retryCount <= 5:
response = self.get_repository_by_name_curl(
component_user, component_user_keytab,
component_user_principal, repo_name, component, 'true')
if response is not None:
Logger.info('{0} Repository {1} exist'.format(
component.title(), (response['name'])))
break
else:
response = self.create_repository_curl(
component_user, component_user_keytab,
component_user_principal, repo_name, repo_data,
policy_user)
if response and len(response) > 0:
Logger.info(
'{0} Repository created in Ranger admin'.
format(component.title()))
break
else:
if retryCount < 5:
time.sleep(30) # delay for 30 seconds
retryCount += 1
else:
Logger.error(
'{0} Repository creation failed in Ranger admin'
.format(component.title()))
break
else:
Logger.error("Connection failed to Ranger Admin !")
def __check_if_client_exists(self, serviceName):
Logger.info("Checking if " + serviceName + " client libraries exist")
if not self.__package_exists(serviceName):
error_msg = serviceName + " client libraries do not exist on the PXF node"
Logger.error(error_msg)
raise Fail(error_msg)
return True
else:
Logger.info('Repository not created')
return False
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
Logger.error(
"Error creating service. Http status code - {0}. \n {1}".
format(e.code, e.read()))
return False
else:
Logger.error("Error creating service. Reason - {0}.".format(
e.reason))
return False
except socket.timeout as e:
Logger.error("Error creating service. Reason - {0}".format(e))
return False
def get_repo(url, name, usernamepassword):
try:
base_url = url + '/service/public/v2/api/service?serviceName=' + name + '&serviceType=kms&isEnabled=true'
request = urllib2.Request(base_url)
base64string = base64.encodestring(usernamepassword).replace('\n', '')
request.add_header("Content-Type", "application/json")
request.add_header("Accept", "application/json")
request.add_header("Authorization", "Basic {0}".format(base64string))
result = urllib2.urlopen(request, timeout=20)
response_code = result.getcode()
response = json.loads(result.read())
if response_code == 200 and len(response) > 0:
def hive_service(name, action='start', upgrade_type=None):
import params
import status_params
if name == 'metastore':
pid_file = status_params.hive_metastore_pid
cmd = format(
"{start_metastore_path} {hive_log_dir}/hive.out {hive_log_dir}/hive.err {pid_file} {hive_server_conf_dir} {hive_log_dir}"
)
elif name == 'hiveserver2':
pid_file = status_params.hive_pid
cmd = format(
"{start_hiveserver2_path} {hive_log_dir}/hive-server2.out {hive_log_dir}/hive-server2.err {pid_file} {hive_server_conf_dir} {hive_log_dir}"
)
if params.security_enabled and check_stack_feature(
StackFeature.HIVE_SERVER2_KERBERIZED_ENV,
params.version_for_stack_feature_checks):
hive_kinit_cmd = format(
"{kinit_path_local} -kt {hive_server2_keytab} {hive_principal}; "
)
Execute(hive_kinit_cmd, user=params.hive_user)
pid = get_user_call_output.get_user_call_output(format("cat {pid_file}"),
user=params.hive_user,
is_checked_call=False)[1]
process_id_exists_command = format(
"ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1")
if action == 'start':
if name == 'hiveserver2':
check_fs_root(params.hive_server_conf_dir, params.execute_path)
daemon_cmd = cmd
hadoop_home = params.hadoop_home
hive_bin = "hive"
# upgrading hiveserver2 (rolling_restart) means that there is an existing,
# de-registering hiveserver2; the pid will still exist, but the new
# hiveserver is spinning up on a new port, so the pid will be re-written
if upgrade_type == UPGRADE_TYPE_ROLLING:
process_id_exists_command = None
if params.version and params.stack_root:
hadoop_home = format("{stack_root}/{version}/hadoop")
hive_bin = os.path.join(params.hive_bin, hive_bin)
Execute(daemon_cmd,
user=params.hive_user,
environment={
'HADOOP_HOME': hadoop_home,
'JAVA_HOME': params.java64_home,
'HIVE_BIN': hive_bin
},
path=params.execute_path,
not_if=process_id_exists_command)
if params.hive_jdbc_driver == "com.mysql.jdbc.Driver" or \
params.hive_jdbc_driver == "org.postgresql.Driver" or \
params.hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver":
validation_called = False
if params.hive_jdbc_target is not None:
validation_called = True
validate_connection(params.hive_jdbc_target, params.hive_lib)
if params.hive2_jdbc_target is not None:
validation_called = True
validate_connection(params.hive2_jdbc_target,
params.hive_server2_hive2_lib)
if not validation_called:
emessage = "ERROR! DB connection check should be executed at least one time!"
Logger.error(emessage)
elif action == 'stop':
daemon_kill_cmd = format("{sudo} kill {pid}")
daemon_hard_kill_cmd = format("{sudo} kill -9 {pid}")
Execute(daemon_kill_cmd,
not_if=format("! ({process_id_exists_command})"))
wait_time = 5
Execute(
daemon_hard_kill_cmd,
not_if=format(
"! ({process_id_exists_command}) || ( sleep {wait_time} && ! ({process_id_exists_command}) )"
),
ignore_failures=True)
try:
# check if stopped the process, else fail the task
Execute(
format("! ({process_id_exists_command})"),
tries=20,
try_sleep=3,
)
except:
show_logs(params.hive_log_dir, params.hive_user)
raise
File(pid_file, action="delete")
run_prepare_war = True
Logger.info(format("Will run prepare war cmd since marker file {prepare_war_cmd_file} has contents which differ.\n" \
"Expected: {command_to_file}.\nActual: {cmd}."))
else:
run_prepare_war = True
Logger.info(format("Will run prepare war cmd since marker file {prepare_war_cmd_file} is missing."))
if run_prepare_war:
# Time-consuming to run
return_code, output = shell.call(command, user=params.oozie_user)
if output is None:
output = ""
if return_code != 0 or "New Oozie WAR file with added".lower() not in output.lower():
message = "Unexpected Oozie WAR preparation output {0}".format(output)
Logger.error(message)
raise Fail(message)
# Generate marker file
File(prepare_war_cmd_file,
content=command_to_file,
mode=0644,
)
else:
Logger.info(format("No need to run prepare-war since marker file {prepare_war_cmd_file} already exists."))
###############################################
# PREPARE-WAR END [BEGIN]
###############################################
oozie_shared_lib = format("/usr/hdp/{hdp_version}/oozie/share")
oozie_user = '******'
oozie_hdfs_user_dir = format("{hdfs_path_prefix}/user/{oozie_user}")
error_message = ""
for check_name in FALLIBLE_CHECKS:
if check_name in structured_output and "exit_code" in structured_output[check_name] \
and structured_output[check_name]["exit_code"] != 0:
error_message += "Check {0} was unsuccessful. Exit code: {1}.".format(check_name, \
structured_output[check_name]["exit_code"])
if "message" in structured_output[check_name]:
error_message += " Message: {0}".format(structured_output[check_name]["message"])
error_message += "\n"
Logger.info("Host checks completed.")
Logger.debug("Structured output: " + str(structured_output))
if error_message:
Logger.error(error_message)
raise Fail(error_message)
def execute_transparent_huge_page_check(self, config):
Logger.info("Transparent huge page check started.")
thp_regex = "\[(.+)\]"
file_name = None
if OSCheck.is_ubuntu_family():
file_name = THP_FILE_UBUNTU
elif OSCheck.is_redhat_family():
file_name = THP_FILE_REDHAT
if file_name and os.path.isfile(file_name):
with open(file_name) as f:
file_content = f.read()
def hive_interactive(name=None):
import params
MB_TO_BYTES = 1048576
# if warehouse directory is in DFS
if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(
params.default_fs).scheme:
# Create Hive Metastore Warehouse Dir
params.HdfsResource(params.hive_metastore_warehouse_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=0700)
# create directories for llap package
pkg_dir = '/user/' + params.hive_user + '/.yarn'
for dir in [pkg_dir, pkg_dir + '/package', pkg_dir + '/package/LLAP']:
# hdfsresouces handles parent creation badly
params.HdfsResource(dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=0755)
if not is_empty(params.tez_hook_proto_base_directory):
params.HdfsResource(params.tez_hook_proto_base_directory,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01755)
if not is_empty(params.hive_hook_proto_base_directory):
params.HdfsResource(params.hive_hook_proto_base_directory,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
dag_meta = params.tez_hook_proto_base_directory + "dag_meta"
params.HdfsResource(dag_meta,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
dag_data = params.tez_hook_proto_base_directory + "dag_data"
params.HdfsResource(dag_data,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
app_data = params.tez_hook_proto_base_directory + "app_data"
params.HdfsResource(app_data,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
else:
Logger.info(
format(
"Not creating warehouse directory '{hive_metastore_warehouse_dir}', as the location is not in DFS."
))
# Create Hive User Dir
params.HdfsResource(params.hive_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=params.hive_hdfs_user_mode)
# list of properties that should be excluded from the config
# this approach is a compromise against adding a dedicated config
# type for hive_server_interactive or needed config groups on a
# per component basis
exclude_list = ['hive.enforce.bucketing', 'hive.enforce.sorting']
# List of configs to be excluded from hive2 client, but present in Hive2 server.
exclude_list_for_hive2_client = [
'javax.jdo.option.ConnectionPassword',
'hadoop.security.credential.provider.path'
]
Logger.info("Directories to fill with configs: %s" %
str(params.hive_conf_dirs_list))
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
'''
As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml,
we need to merge hive/hive-site.xml and hive2/hive-site.xml and store it in hive2/hive-site.xml.
'''
merged_hive_interactive_site = {}
merged_hive_interactive_site.update(params.hive_site_config)
merged_hive_interactive_site.update(
params.config['configurations']['hive-interactive-site'])
for item in exclude_list:
if item in merged_hive_interactive_site.keys():
del merged_hive_interactive_site[item]
merged_hive_interactive_site[
'hive.llap.daemon.vcpus.per.instance'] = format(
merged_hive_interactive_site['hive.llap.daemon.vcpus.per.instance']
)
merged_hive_interactive_site[
'hive.server2.active.passive.ha.enable'] = str(
params.hive_server_interactive_ha).lower()
'''
Config 'hive.llap.io.memory.size' calculated value in stack_advisor is in MB as of now. We need to
convert it to bytes before we write it down to config file.
'''
if 'hive.llap.io.memory.size' in merged_hive_interactive_site.keys():
hive_llap_io_mem_size_in_mb = merged_hive_interactive_site.get(
"hive.llap.io.memory.size")
hive_llap_io_mem_size_in_bytes = long(
hive_llap_io_mem_size_in_mb) * MB_TO_BYTES
merged_hive_interactive_site[
'hive.llap.io.memory.size'] = hive_llap_io_mem_size_in_bytes
Logger.info(
"Converted 'hive.llap.io.memory.size' value from '{0} MB' to '{1} Bytes' before writing "
"it to config file.".format(hive_llap_io_mem_size_in_mb,
hive_llap_io_mem_size_in_bytes))
'''
Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook',
which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed.
'''
# Generate atlas-application.properties.xml file
if params.enable_atlas_hook and params.stack_supports_atlas_hook_for_hive_interactive:
Logger.info("Setup for Atlas Hive2 Hook started.")
atlas_hook_filepath = os.path.join(
params.hive_server_interactive_conf_dir,
params.atlas_hook_filename)
setup_atlas_hook(SERVICE.HIVE,
params.hive_atlas_application_properties,
atlas_hook_filepath, params.hive_user,
params.user_group)
Logger.info("Setup for Atlas Hive2 Hook done.")
else:
# Required for HDP 2.5 stacks
Logger.info(
"Skipping setup for Atlas Hook, as it is disabled/ not supported.")
remove_atlas_hook_if_exists(merged_hive_interactive_site)
'''
As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml,
we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml.
'''
merged_tez_interactive_site = {}
if 'tez-site' in params.config['configurations']:
merged_tez_interactive_site.update(
params.config['configurations']['tez-site'])
Logger.info(
"Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'."
)
else:
Logger.error(
"Tez's 'tez-site' couldn't be retrieved from passed-in configurations."
)
merged_tez_interactive_site.update(
params.config['configurations']['tez-interactive-site'])
XmlConfig("tez-site.xml",
conf_dir=params.tez_interactive_conf_dir,
configurations=merged_tez_interactive_site,
configuration_attributes=params.config['configurationAttributes']
['tez-interactive-site'],
owner=params.tez_interactive_user,
group=params.user_group,
mode=0664)
'''
Merge properties from hiveserver2-interactive-site into hiveserver2-site
'''
merged_hiveserver2_interactive_site = {}
if 'hiveserver2-site' in params.config['configurations']:
merged_hiveserver2_interactive_site.update(
params.config['configurations']['hiveserver2-site'])
Logger.info(
"Retrieved 'hiveserver2-site' for merging with 'hiveserver2-interactive-site'."
)
else:
Logger.error(
"'hiveserver2-site' couldn't be retrieved from passed-in configurations."
)
merged_hiveserver2_interactive_site.update(
params.config['configurations']['hiveserver2-interactive-site'])
# Create config files under hive_server_interactive_conf_dir:
# hive-site.xml
# hive-env.sh
# llap-daemon-log4j2.properties
# llap-cli-log4j2.properties
# hive-log4j2.properties
# hive-exec-log4j2.properties
# beeline-log4j2.properties
hive_server_interactive_conf_dir = params.hive_server_interactive_conf_dir
mode_identified = 0600
merged_hive_interactive_site = update_credential_provider_path(
merged_hive_interactive_site, 'hive-site',
os.path.join(conf_dir, 'hive-site.jceks'), params.hive_user,
params.user_group)
XmlConfig("hive-site.xml",
conf_dir=hive_server_interactive_conf_dir,
configurations=merged_hive_interactive_site,
configuration_attributes=params.config['configurationAttributes']
['hive-interactive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
XmlConfig("hiveserver2-site.xml",
conf_dir=hive_server_interactive_conf_dir,
configurations=merged_hiveserver2_interactive_site,
configuration_attributes=params.config['configurationAttributes']
['hiveserver2-interactive-site'],
owner=params.hive_user,
group=params.user_group,
mode=mode_identified)
File(format("{hive_server_interactive_conf_dir}/hive-env.sh"),
owner=params.hive_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.hive_interactive_env_sh_template))
llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.llap_daemon_log4j))
llap_cli_log4j2_filename = 'llap-cli-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.llap_cli_log4j2))
hive_log4j2_filename = 'hive-log4j2.properties'
File(format("{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.hive_log4j2))
hive_exec_log4j2_filename = 'hive-exec-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.hive_exec_log4j2))
beeline_log4j2_filename = 'beeline-log4j2.properties'
File(
format("{hive_server_interactive_conf_dir}/{beeline_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.beeline_log4j2))
XmlConfig("beeline-site.xml",
conf_dir=conf_dir,
configurations=params.beeline_site_config,
owner=params.hive_user,
group=params.user_group,
mode=mode_identified)
File(os.path.join(hive_server_interactive_conf_dir,
"hadoop-metrics2-hiveserver2.properties"),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=Template("hadoop-metrics2-hiveserver2.properties.j2"))
File(format(
"{hive_server_interactive_conf_dir}/hadoop-metrics2-llapdaemon.properties"
),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=Template("hadoop-metrics2-llapdaemon.j2"))
File(format(
"{hive_server_interactive_conf_dir}/hadoop-metrics2-llaptaskscheduler.properties"
),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=Template("hadoop-metrics2-llaptaskscheduler.j2"))
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'hive.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hive.conf.j2"))
if not os.path.exists(params.target_hive_interactive):
jdbc_connector(params.target_hive_interactive,
params.hive_intaractive_previous_jdbc_jar)
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}/{check_db_connection_jar_name}")),
mode=0644)
File(params.start_hiveserver2_interactive_path,
mode=0755,
content=Template(format('{start_hiveserver2_interactive_script}')))
Directory(params.hive_pid_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_log_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_interactive_var_lib,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
generate_logfeeder_input_config(
'hive', Template("input.config-hive.json.j2", extra_imports=[default]))
def _llap_start(self, env, cleanup=False):
import params
env.set_params(params)
Logger.info("Starting LLAP")
# TODO, start only if not already running.
# TODO : Currently hardcoded the params. Need to read the suggested values from hive2/hive-site.xml.
# TODO, ensure that script works as hive from cmd when not cd'ed in /homve/hive
# Needs permission to write to hive home dir.
cmd = ''
if params.security_enabled:
cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances 1 -slider-am-container-mb "
"{slider_am_container_mb} --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab "
"{hive_llap_keytab_file} --slider-principal {hive_headless_keytab} --loglevel INFO")
else:
cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances 1 -slider-am-container-mb {slider_am_container_mb} --loglevel INFO")
run_file_path = None
try:
Logger.info(format("Command: {cmd}"))
cmd = cmd.split()
code, output, error = shell.checked_call(cmd, user=params.hive_user, stderr=subprocess.PIPE, logoutput=True)
if code != 0 or output is None:
raise Fail("Command failed with either non-zero return code or no output.")
# E.g., output:
# Prepared llap-slider-05Apr2016/run.sh for running LLAP on Slider
exp = r"Prepared (.*?run.sh) for running LLAP"
m = re.match(exp, output, re.I)
if m and len(m.groups()) == 1:
run_file_name = m.group(1)
run_file_path = os.path.join(params.hive_user_home_dir, run_file_name)
else:
raise Fail("Did not find run.sh file in output: " + str(output))
Logger.info(format("Run file path: {run_file_path}"))
if os.path.isfile(run_file_path):
Execute(run_file_path, user=params.hive_user)
# TODO : Sleep below is not a good idea. We need to check the status of LLAP app to figure out it got
# launched properly and is in running state. Then go ahead with Hive Interactive Server start.
Logger.info("Sleeping for 30 secs")
time.sleep(30)
Logger.info("LLAP app deployed successfully.")
return True
else:
raise Fail(format("Did not find run file {run_file_path}"))
except:
# Attempt to clean up the packaged application, or potentially rename it with a .bak
if run_file_path is not None and cleanup:
try:
parent_dir = os.path.dirname(run_file_path)
if os.path.isdir(parent_dir):
shutil.rmtree(parent_dir)
except Exception, e:
Logger.error("Could not cleanup LLAP app package. Error: " + str(e))
# throw the original exception
raise
def setup_ranger_plugin(component_select_name,
service_name,
previous_jdbc_jar,
component_downloaded_custom_connector,
component_driver_curl_source,
component_driver_curl_target,
java_home,
repo_name,
plugin_repo_dict,
ranger_env_properties,
plugin_properties,
policy_user,
policymgr_mgr_url,
plugin_enabled,
conf_dict,
component_user,
component_group,
cache_service_list,
plugin_audit_properties,
plugin_audit_attributes,
plugin_security_properties,
plugin_security_attributes,
plugin_policymgr_ssl_properties,
plugin_policymgr_ssl_attributes,
component_list,
audit_db_is_enabled,
credential_file,
xa_audit_db_password,
ssl_truststore_password,
ssl_keystore_password,
api_version=None,
stack_version_override=None,
skip_if_rangeradmin_down=True,
is_security_enabled=False,
is_stack_supports_ranger_kerberos=False,
component_user_principal=None,
component_user_keytab=None):
if audit_db_is_enabled and component_driver_curl_source is not None and not component_driver_curl_source.endswith(
"/None"):
if previous_jdbc_jar and os.path.isfile(previous_jdbc_jar):
File(previous_jdbc_jar, action='delete')
File(component_downloaded_custom_connector,
content=DownloadSource(component_driver_curl_source),
mode=0644)
Execute(('cp', '--remove-destination',
component_downloaded_custom_connector,
component_driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(component_driver_curl_target, mode=0644)
if policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
stack_version = get_stack_version(component_select_name)
if stack_version_override is not None:
stack_version = stack_version_override
component_conf_dir = conf_dict
if plugin_enabled:
service_name_exist = False
policycache_path = os.path.join('/etc', 'ranger', repo_name,
'policycache')
try:
for cache_service in cache_service_list:
policycache_json_file = format(
'{policycache_path}/{cache_service}_{repo_name}.json')
if os.path.isfile(policycache_json_file) and os.path.getsize(
policycache_json_file) > 0:
with open(policycache_json_file) as json_file:
json_data = json.load(json_file)
if 'serviceName' in json_data and json_data[
'serviceName'] == repo_name:
service_name_exist = True
Logger.info(
"Skipping Ranger API calls, as policy cache file exists for {0}"
.format(service_name))
break
except Exception, err:
Logger.error(
"Error occurred while fetching service name from policy cache file.\nError: {0}"
.format(err))
if not service_name_exist:
if api_version is not None and api_version == 'v2':
ranger_adm_obj = RangeradminV2(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(
service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'],
ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'],
ranger_env_properties['admin_password'], policy_user,
is_security_enabled, is_stack_supports_ranger_kerberos,
component_user, component_user_principal,
component_user_keytab)
else:
ranger_adm_obj = Rangeradmin(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(
service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'],
ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'],
ranger_env_properties['admin_password'], policy_user)
current_datetime = datetime.now()
File(
format('{component_conf_dir}/ranger-security.xml'),
owner=component_user,
group=component_group,
mode=0644,
content=InlineTemplate(
format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
)))
Directory([
os.path.join('/etc', 'ranger', repo_name),
os.path.join('/etc', 'ranger', repo_name, 'policycache')
],
owner=component_user,
group=component_group,
mode=0775,
create_parents=True,
cd_access='a')
for cache_service in cache_service_list:
File(os.path.join('/etc', 'ranger', repo_name, 'policycache',
format('{cache_service}_{repo_name}.json')),
owner=component_user,
group=component_group,
mode=0644)
# remove plain-text password from xml configs
plugin_audit_password_property = 'xasecure.audit.destination.db.password'
plugin_audit_properties_copy = {}
plugin_audit_properties_copy.update(plugin_audit_properties)
if plugin_audit_password_property in plugin_audit_properties_copy:
plugin_audit_properties_copy[
plugin_audit_password_property] = "crypted"
XmlConfig(format('ranger-{service_name}-audit.xml'),
conf_dir=component_conf_dir,
configurations=plugin_audit_properties_copy,
configuration_attributes=plugin_audit_attributes,
owner=component_user,
group=component_group,
mode=0744)
XmlConfig(format('ranger-{service_name}-security.xml'),
conf_dir=component_conf_dir,
configurations=plugin_security_properties,
configuration_attributes=plugin_security_attributes,
owner=component_user,
group=component_group,
mode=0744)
# remove plain-text password from xml configs
plugin_password_properties = [
'xasecure.policymgr.clientssl.keystore.password',
'xasecure.policymgr.clientssl.truststore.password'
]
plugin_policymgr_ssl_properties_copy = {}
plugin_policymgr_ssl_properties_copy.update(
plugin_policymgr_ssl_properties)
for prop in plugin_password_properties:
if prop in plugin_policymgr_ssl_properties_copy:
plugin_policymgr_ssl_properties_copy[prop] = "crypted"
if str(service_name).lower() == 'yarn':
XmlConfig("ranger-policymgr-ssl-yarn.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties_copy,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
else:
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties_copy,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
# creating symblink should be done by rpm package
# setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list)
setup_ranger_plugin_keystore(service_name, audit_db_is_enabled,
stack_version, credential_file,
xa_audit_db_password,
ssl_truststore_password,
ssl_keystore_password, component_user,
component_group, java_home)
def setup_ranger_hbase():
import params
if params.has_ranger_admin:
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source)
)
if not os.path.isfile(params.driver_curl_target):
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
try:
command = 'hdp-select status hbase-client'
return_code, hdp_output = shell.call(command, timeout=20)
except Exception, e:
Logger.error(str(e))
raise Fail('Unable to execute hdp-select command to retrieve the version.')
if return_code != 0:
raise Fail('Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))
hdp_version = re.sub('hbase-client - ', '', hdp_output).strip()
match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
if match is None:
raise Fail('Failed to get extracted version')
file_path = '/usr/hdp/'+ hdp_version +'/ranger-hbase-plugin/install.properties'
if not os.path.isfile(file_path):
raise Fail('Ranger HBase plugin install.properties file does not exist at {0}'.format(file_path))
ranger_hbase_dict = ranger_hbase_properties()
hbase_repo_data = hbase_repo_properties()
write_properties_to_file(file_path, ranger_hbase_dict)
if params.enable_ranger_hbase:
cmd = format('cd /usr/hdp/{hdp_version}/ranger-hbase-plugin/ && sh enable-hbase-plugin.sh')
ranger_adm_obj = Rangeradmin(url=ranger_hbase_dict['POLICY_MGR_URL'])
response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(ranger_hbase_dict['POLICY_MGR_URL'] + '/login.jsp', 'test:test')
if response_code is not None and response_code == 200:
ambari_ranger_admin, ambari_ranger_password = ranger_adm_obj.create_ambari_admin_user(params.ambari_ranger_admin, params.ambari_ranger_password, params.admin_uname_password)
ambari_username_password_for_ranger = ambari_ranger_admin + ':' + ambari_ranger_password
if ambari_ranger_admin != '' and ambari_ranger_password != '':
repo = ranger_adm_obj.get_repository_by_name_urllib2(ranger_hbase_dict['REPOSITORY_NAME'], 'hbase', 'true', ambari_username_password_for_ranger)
if repo and repo['name'] == ranger_hbase_dict['REPOSITORY_NAME']:
Logger.info('Hbase Repository exist')
else:
response = ranger_adm_obj.create_repository_urllib2(hbase_repo_data, ambari_username_password_for_ranger, params.policy_user)
if response is not None:
Logger.info('Hbase Repository created in Ranger admin')
else:
Logger.info('Hbase Repository creation failed in Ranger admin')
else:
Logger.info('Ambari admin username and password are blank ')
else:
Logger.info('Ranger service is not started on given host')
else:
cmd = format('cd /usr/hdp/{hdp_version}/ranger-hbase-plugin/ && sh disable-hbase-plugin.sh')
Execute(cmd, environment={'JAVA_HOME': params.java64_home}, logoutput=True)
def hive_service_interactive(name, action='start'):
import params
import status_params
pid_file = status_params.hive_interactive_pid
cmd = format(
"{start_hiveserver2_interactive_path} {hive_pid_dir}/hive-server2-interactive.out {hive_log_dir}/hive-server2-interactive.err {pid_file} {hive_server_interactive_conf_dir} {tez_interactive_conf_dir}"
)
pid = get_user_call_output.get_user_call_output(format("cat {pid_file}"),
user=params.hive_user,
is_checked_call=False)[1]
process_id_exists_command = format(
"ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1")
if action == 'start':
check_fs_root(params.hive_server_interactive_conf_dir,
params.execute_path_hive_interactive)
daemon_cmd = cmd
hadoop_home = params.hadoop_home
hive_interactive_bin = "hive2"
Execute(daemon_cmd,
user=params.hive_user,
environment={
'HADOOP_HOME': hadoop_home,
'JAVA_HOME': params.java64_home,
'HIVE_BIN': hive_interactive_bin
},
path=params.execute_path,
not_if=process_id_exists_command)
if params.hive_jdbc_driver == "com.mysql.jdbc.Driver" or \
params.hive_jdbc_driver == "org.postgresql.Driver" or \
params.hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver":
path_to_jdbc = params.target_hive_interactive
if not params.jdbc_jar_name:
path_to_jdbc = format("{hive_interactive_lib}/") + \
params.default_connectors_map[params.hive_jdbc_driver] if params.hive_jdbc_driver in params.default_connectors_map else None
if not os.path.isfile(path_to_jdbc):
path_to_jdbc = format("{hive_interactive_lib}/") + "*"
error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \
" in hive lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'"
Logger.error(error_message)
db_connection_check_command = format(
"{java64_home}/bin/java -cp {check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{hive_jdbc_connection_url}' {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}"
)
Execute(db_connection_check_command,
path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
tries=5,
try_sleep=10)
elif action == 'stop':
daemon_kill_cmd = format("{sudo} kill {pid}")
daemon_hard_kill_cmd = format("{sudo} kill -9 {pid}")
Execute(daemon_kill_cmd,
not_if=format("! ({process_id_exists_command})"))
# check if stopped the process, otherwise send hard kill command.
try:
Execute(
format("! ({process_id_exists_command})"),
tries=10,
try_sleep=3,
)
except:
Execute(daemon_hard_kill_cmd,
not_if=format("! ({process_id_exists_command}) "))
# check if stopped the process, else fail the task
Execute(
format("! ({process_id_exists_command})"),
tries=20,
try_sleep=3,
)
File(pid_file, action="delete")
