def service(component_name, action="start"):
import params
if component_name.lower() == "mesos_dispatcher":
daemon = params.spark_dispatch_start
pid_file = params.mesos_dispatcher_pid_file
usr = params.spark_user
else:
# daemon = params.spark_history_server_start
pass
if action == "start":
check_process = format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1")
cmd = format("export MESOS_NATIVE_JAVA_LIBRARY={mesos_native_java_library} && " "{daemon}")
# Remove the pid file if its corresponding process is not running.
File(pid_file, action="delete", not_if=check_process)
# Attempt to start the process. Internally, this is skipped if the process is already running.
Execute(cmd, user=usr, environment={"JAVA_HOME": params.java_home}, not_if=check_process)
# Ensure that the process with the expected PID exists.
Execute(check_process, user=usr, not_if=check_process, initial_wait=5)
elif action == "stop":
cmd = format("{spark_dispatch_stop}")
Execute(cmd, user=usr, environment={"JAVA_HOME": params.java_home})
File(pid_file, action="delete")
def jdbc_connector():
import params
from urllib2 import HTTPError
from resource_management import Fail
for jar_name in params.sqoop_jdbc_drivers_dict:
if 'mysql-connector-java.jar' in jar_name:
continue
downloaded_custom_connector = format("{sqoop_lib}/{jar_name}")
jdbc_symlink_remote = params.sqoop_jdbc_drivers_dict[jar_name]
jdbc_driver_label = params.sqoop_jdbc_drivers_name_dict[jar_name]
driver_curl_source = format("{jdk_location}/{jdbc_symlink_remote}")
environment = {
"no_proxy": format("{ambari_server_hostname}")
}
try:
File(downloaded_custom_connector,
content = DownloadSource(driver_curl_source),
mode = 0644,
)
except HTTPError:
error_string = format("Could not download {driver_curl_source}\n\
Please upload jdbc driver to server by run command:\n\
ambari-server setup --jdbc-db={jdbc_driver_label} --jdbc-driver=<PATH TO DRIVER>\n\
at {ambari_server_hostname}")
raise Fail(error_string)
def _get_current_hiveserver_version():
"""
Runs "hive --version" and parses the result in order
to obtain the current version of hive.
:return: the hiveserver2 version, returned by "hive --version"
"""
import params
try:
# When downgrading the source version should be the version we are downgrading from
if "downgrade" == params.upgrade_direction:
if not params.downgrade_from_version:
raise Fail('The version from which we are downgrading from should be provided in \'downgrade_from_version\'')
source_version = params.downgrade_from_version
else:
source_version = params.current_version
hive_execute_path = _get_hive_execute_path(source_version)
version_hive_bin = params.hive_bin
formatted_source_version = format_hdp_stack_version(source_version)
if formatted_source_version and compare_versions(formatted_source_version, "2.2") >= 0:
version_hive_bin = format('/usr/hdp/{source_version}/hive/bin')
command = format('{version_hive_bin}/hive --version')
return_code, hdp_output = shell.call(command, user=params.hive_user, path=hive_execute_path)
except Exception, e:
Logger.error(str(e))
raise Fail('Unable to execute hive --version command to retrieve the hiveserver2 version.')
def spark_service(action):
import params
if action == 'start':
if params.security_enabled:
spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
Execute(spark_kinit_cmd, user=params.spark_user)
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
if params.hdp_stack_version and compare_versions(params.hdp_stack_version, '2.3.0.0') < 0:
resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user)
if resource_created:
params.HdfsResource(None, action="execute")
no_op_test = format(
'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1')
Execute(format('{spark_history_server_start}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home},
not_if=no_op_test
)
elif action == 'stop':
Execute(format('{spark_history_server_stop}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home}
)
File(params.spark_history_server_pid_file,
action="delete"
)
def _get_directory_mappings_during_upgrade():
"""
Gets a dictionary of directory to archive name that represents the
directories that need to be backed up and their output tarball archive targets
:return: the dictionary of directory to tarball mappings
"""
import params
# Must be performing an Upgrade
if params.upgrade_direction is None or params.upgrade_direction != Direction.UPGRADE or \
params.upgrade_from_version is None or params.upgrade_from_version == "":
Logger.error("Function _get_directory_mappings_during_upgrade() can only be called during a Stack Upgrade in direction UPGRADE.")
return {}
# By default, use this for all stacks.
knox_data_dir = '/var/lib/knox/data'
if params.stack_name and params.stack_name.upper() == "HDP" and \
compare_versions(format_hdp_stack_version(params.upgrade_from_version), "2.3.0.0") > 0:
# Use the version that is being upgraded from.
knox_data_dir = format('/usr/hdp/{upgrade_from_version}/knox/data')
# the trailing "/" is important here so as to not include the "conf" folder itself
directories = {knox_data_dir: BACKUP_DATA_ARCHIVE, params.knox_conf_dir + "/": BACKUP_CONF_ARCHIVE}
Logger.info(format("Knox directories to backup:\n{directories}"))
return directories
def _create_file(self, target, source=None, mode=""):
"""
PUT file command in slow, however _get_file_status is pretty fast,
so we should check if the file really should be put before doing it.
"""
file_status = self._get_file_status(target) if target!=self.main_resource.resource.target else self.target_status
mode = "" if not mode else mode
if file_status:
if source:
length = file_status['length']
local_file_size = os.stat(source).st_size # TODO: os -> sudo
# TODO: re-implement this using checksums
if local_file_size == length:
Logger.info(format("DFS file {target} is identical to {source}, skipping the copying"))
return
elif not self.main_resource.resource.replace_existing_files:
Logger.info(format("Not replacing existing DFS file {target} which is different from {source}, due to replace_existing_files=False"))
return
else:
Logger.info(format("File {target} already exists in DFS, skipping the creation"))
return
Logger.info(format("Creating new file {target} in DFS"))
kwargs = {'permission': mode} if mode else {}
self.util.run_command(target, 'CREATE', method='PUT', overwrite=True, assertable_result=False, file_to_put=source, **kwargs)
if mode and file_status:
file_status['permission'] = mode
def check_thrift_port_sasl(address, port, hive_auth = "NOSASL", key = None, kinitcmd = None, smokeuser = '******',
transport_mode = "binary"):
"""
Hive thrift SASL port check
"""
BEELINE_CHECK_TIMEOUT = 30
if kinitcmd:
url = format("jdbc:hive2://{address}:{port}/;principal={key}")
Execute(kinitcmd,
user=smokeuser
)
else:
url = format("jdbc:hive2://{address}:{port}")
if hive_auth != "NOSASL" and transport_mode != "http":
cmd = format("! beeline -u '{url}' -e '' ") + "2>&1| awk '{print}'|grep -i -e 'Connection refused' -e 'Invalid URL'"
Execute(cmd,
user=smokeuser,
path=["/bin/", "/usr/bin/", "/usr/lib/hive/bin/", "/usr/sbin/"],
timeout=BEELINE_CHECK_TIMEOUT
)
else:
s = socket.socket()
s.settimeout(1)
try:
s.connect((address, port))
except socket.error, e:
raise
finally:
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations (dictionary): a mapping of configuration key to value
parameters (dictionary): a mapping of script parameter key to value
host_name (string): the name of this host where the alert is running
"""
from resource_management.libraries.functions import reload_windows_env
from resource_management.core.resources import Execute
reload_windows_env()
hive_home = os.environ['HIVE_HOME']
if configurations is None:
return ('UNKNOWN', ['There were no configurations supplied to the script.'])
transport_mode = HIVE_SERVER_TRANSPORT_MODE_DEFAULT
if HIVE_SERVER_TRANSPORT_MODE_KEY in configurations:
transport_mode = configurations[HIVE_SERVER_TRANSPORT_MODE_KEY]
port = THRIFT_PORT_DEFAULT
if transport_mode.lower() == 'binary' and HIVE_SERVER_THRIFT_PORT_KEY in configurations:
port = int(configurations[HIVE_SERVER_THRIFT_PORT_KEY])
elif transport_mode.lower() == 'http' and HIVE_SERVER_THRIFT_HTTP_PORT_KEY in configurations:
port = int(configurations[HIVE_SERVER_THRIFT_HTTP_PORT_KEY])
hiveuser = HADOOPUSER_DEFAULT
if HADOOPUSER_KEY in configurations:
hiveuser = configurations[HADOOPUSER_KEY]
result_code = None
try:
if host_name is None:
host_name = socket.getfqdn()
beeline_url = ['jdbc:hive2://{host_name}:{port}/', "transportMode={transport_mode}"]
# append url according to used transport
if transport_mode == "http":
beeline_url.append('httpPath=cliservice')
beeline_url_string = format(";".join(beeline_url))
beeline_cmd = os.path.join(hive_home, "bin", "beeline.cmd")
cmd = format("cmd /c {beeline_cmd} -u {beeline_url_string} -e '' 2>&1 | findstr Connected")
start_time = time.time()
try:
Execute(cmd, user=hiveuser, timeout=30)
total_time = time.time() - start_time
result_code = 'OK'
label = OK_MESSAGE.format(total_time, port)
except:
result_code = 'CRITICAL'
label = CRITICAL_MESSAGE.format(host_name, port, traceback.format_exc())
except:
label = traceback.format_exc()
result_code = 'UNKNOWN'
return (result_code, [label])
def stop(self, env, rolling_restart=False):
import params
env.set_params(params)
daemon_cmd = format('source {params.conf_dir}/atlas-env.sh; {params.metadata_stop_script}')
Execute(daemon_cmd,
user=params.metadata_user,
)
Execute (format("rm -f {params.pid_file}"))
def get_check_command(oozie_url, host_name, configurations):
if OOZIE_USER in configurations:
oozie_user = configurations[OOZIE_USER]
else:
raise Exception("Oozie user is required")
security_enabled = False
if SECURITY_ENABLED in configurations:
security_enabled = str(configurations[SECURITY_ENABLED]).upper() == 'TRUE'
kerberos_env = None
if security_enabled:
if OOZIE_KEYTAB in configurations and OOZIE_PRINCIPAL in configurations:
oozie_keytab = configurations[OOZIE_KEYTAB]
oozie_principal = configurations[OOZIE_PRINCIPAL]
# substitute _HOST in kerberos principal with actual fqdn
oozie_principal = oozie_principal.replace('_HOST', host_name)
else:
raise KerberosPropertiesNotFound('The Oozie keytab and principal are required configurations when security is enabled.')
# Create the kerberos credentials cache (ccache) file and set it in the environment to use
# when executing curl
env = Environment.get_instance()
ccache_file = "{0}{1}oozie_alert_cc_{2}".format(env.tmp_dir, os.sep, os.getpid())
kerberos_env = {'KRB5CCNAME': ccache_file}
# Get the configured Kerberos executable search paths, if any
if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]
else:
kerberos_executable_search_paths = None
klist_path_local = get_klist_path(kerberos_executable_search_paths)
klist_command = format("{klist_path_local} -s {ccache_file}")
# Determine if we need to kinit by testing to see if the relevant cache exists and has
# non-expired tickets. Tickets are marked to expire after 5 minutes to help reduce the number
# it kinits we do but recover quickly when keytabs are regenerated
return_code, _ = call(klist_command, user=oozie_user)
if return_code != 0:
kinit_path_local = get_kinit_path(kerberos_executable_search_paths)
kinit_command = format("{kinit_path_local} -l 5m -kt {oozie_keytab} {oozie_principal}; ")
# kinit
Execute(kinit_command,
environment=kerberos_env,
user=oozie_user,
)
# oozie configuration directory uses a symlink when > HDP 2.2
oozie_config_directory = OOZIE_CONF_DIR_LEGACY
if os.path.exists(OOZIE_CONF_DIR):
oozie_config_directory = OOZIE_CONF_DIR
command = "source {0}/oozie-env.sh ; oozie admin -oozie {1} -status".format(
oozie_config_directory, oozie_url)
return (command, kerberos_env, oozie_user)
def _copy_from_local_directory(self, target, source):
for next_path_part in os.listdir(source):
new_source = os.path.join(source, next_path_part)
new_target = format("{target}/{next_path_part}")
if os.path.isdir(new_source):
Logger.info(format("Creating DFS directory {new_target}"))
self._create_directory(new_target)
self._copy_from_local_directory(new_target, new_source)
else:
self._create_file(new_target, new_source)
def startdemoldap(self, env):
import params
env.set_params(params)
self.configureldap(env)
daemon_cmd = format('{ldap_bin} start')
no_op_test = format('ls {ldap_pid_file} >/dev/null 2>&1 && ps -p `cat {ldap_pid_file}` >/dev/null 2>&1')
Execute(daemon_cmd,
user=params.knox_user,
environment={'JAVA_HOME': params.java_home},
not_if=no_op_test
)
def start(self, env, rolling_restart=False):
import params
env.set_params(params)
self.configure(env)
daemon_cmd = format('source {params.conf_dir}/atlas-env.sh ; {params.metadata_start_script}')
no_op_test = format('ls {params.pid_file} >/dev/null 2>&1 && ps -p `cat {params.pid_file}` >/dev/null 2>&1')
Execute(daemon_cmd,
user=params.metadata_user,
not_if=no_op_test
)
def phoenix_service(action = 'start'): # 'start', 'stop', 'status'
# Note: params/status_params should already be imported before calling phoenix_service()
pid_file = format("{pid_dir}/phoenix-{hbase_user}-server.pid")
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1")
if action == "status":
check_process_status(pid_file)
else:
env = {'JAVA_HOME': format("{java64_home}"), 'HBASE_CONF_DIR': format("{hbase_conf_dir}")}
daemon_cmd = format("{phx_daemon_script} {action}")
if action == 'start':
Execute(daemon_cmd,
user=format("{hbase_user}"),
environment=env)
elif action == 'stop':
Execute(daemon_cmd,
timeout = 30,
on_timeout = format("! ( {no_op_test} ) || {sudo} -H -E kill -9 `cat {pid_file}`"),
user=format("{hbase_user}"),
environment=env
)
File(pid_file,
action = "delete"
)
def run_migration(env, upgrade_type):
"""
If the acl migration script is present, then run it for either upgrade or downgrade.
That script was introduced in HDP 2.3.4.0 and requires stopping all Kafka brokers first.
Requires configs to be present.
:param env: Environment.
:param upgrade_type: "rolling" or "nonrolling
"""
import params
if upgrade_type is None:
raise Fail('Parameter "upgrade_type" is missing.')
if params.upgrade_direction is None:
raise Fail('Parameter "upgrade_direction" is missing.')
if params.upgrade_direction == Direction.DOWNGRADE and params.downgrade_from_version is None:
raise Fail('Parameter "downgrade_from_version" is missing.')
if not params.security_enabled:
Logger.info("Skip running the Kafka ACL migration script since cluster security is not enabled.")
return
Logger.info("Upgrade type: {0}, direction: {1}".format(str(upgrade_type), params.upgrade_direction))
# If the schema upgrade script exists in the version upgrading to, then attempt to upgrade/downgrade it while still using the present bits.
kafka_acls_script = None
command_suffix = ""
if params.upgrade_direction == Direction.UPGRADE:
kafka_acls_script = format("/usr/hdp/{version}/kafka/bin/kafka-acls.sh")
command_suffix = "--upgradeAcls"
elif params.upgrade_direction == Direction.DOWNGRADE:
kafka_acls_script = format("/usr/hdp/{downgrade_from_version}/kafka/bin/kafka-acls.sh")
command_suffix = "--downgradeAcls"
if kafka_acls_script is not None:
if os.path.exists(kafka_acls_script):
Logger.info("Found Kafka acls script: {0}".format(kafka_acls_script))
if params.zookeeper_connect is None:
raise Fail("Could not retrieve property kafka-broker/zookeeper.connect")
acls_command = "{0} --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect={1} {2}".\
format(kafka_acls_script, params.zookeeper_connect, command_suffix)
Execute(acls_command,
user=params.kafka_user,
logoutput=True)
else:
Logger.info("Did not find Kafka acls script: {0}".format(kafka_acls_script))
def get_check_command(oozie_url, host_name, configurations):
from resource_management.libraries.functions import reload_windows_env
reload_windows_env()
oozie_home = os.environ['OOZIE_HOME']
oozie_cmd = os.path.join(oozie_home, 'bin', 'oozie.cmd')
command = format("cmd /c {oozie_cmd} admin -oozie {oozie_url} -status")
return (command, None, None)
def install_ranger_files(self):
import params
splice_lib_dir = "/var/lib/splicemachine"
ranger_home = format('{params.stack_root}/current/ranger-admin')
if not os.path.exists(ranger_home):
print("No ranger installation found")
return
ranger_user = params.config['configurations']['ranger-env']['ranger_user']
ranger_plugins_dir = os.path.join(ranger_home,
"ews/webapp/WEB-INF/classes/ranger-plugins/splicemachine")
Directory(ranger_plugins_dir,
owner = ranger_user,
group = ranger_user,
create_parents = False
)
splice_ranger_jar = self.search_file(splice_lib_dir, "splice_ranger_admin*.jar")
db_client_jar = self.search_file(splice_lib_dir, "db-client-*.jar")
Link(os.path.join(ranger_plugins_dir, splice_ranger_jar),
to = os.path.join(splice_lib_dir, splice_ranger_jar))
Link(os.path.join(ranger_plugins_dir, db_client_jar),
to = os.path.join(splice_lib_dir, db_client_jar))
hbase_user = params.config['configurations']['hbase-env']['hbase_user']
hdfs_audit_dir = params.config['configurations']['ranger-splicemachine-audit'][
'xasecure.audit.destination.hdfs.dir']
params.HdfsResource(hdfs_audit_dir,
type="directory",
action="create_on_execute",
owner=hbase_user
)
def pre_rolling_restart(self, env):
"""
Performs the tasks surrounding the Oozie startup when a rolling upgrade
is in progress. This includes backing up the configuration, updating
the database, preparing the WAR, and installing the sharelib in HDFS.
:param env:
:return:
"""
import params
env.set_params(params)
# this function should not execute if the version can't be determined or
# is not at least HDP 2.2.0.0
if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0:
return
Logger.info("Executing Oozie Server Rolling Upgrade pre-restart")
oozie_server_upgrade.backup_configuration()
Execute(format("hdp-select set oozie-server {version}"))
oozie_server_upgrade.restore_configuration()
oozie_server_upgrade.prepare_libext_directory()
oozie_server_upgrade.upgrade_oozie()
def status(self, env):
import status_params
env.set_params(status_params)
pid_file = format("{hive_pid_dir}/{hive_pid}")
# Recursively check all existing gmetad pid files
check_process_status(pid_file)
def get_live_status(pid_file, flume_conf_directory):
"""
Gets the status information of a flume agent, including source, sink, and
channel counts.
:param pid_file: the PID file of the agent to check
:param flume_conf_directory: the configuration directory (ie /etc/flume/conf)
:return: a dictionary of information about the flume agent
"""
pid_file_part = pid_file.split(os.sep).pop()
res = {}
res['name'] = pid_file_part
if pid_file_part.endswith(".pid"):
res['name'] = pid_file_part[:-4]
res['status'] = 'RUNNING' if is_flume_process_live(pid_file) else 'NOT_RUNNING'
res['sources_count'] = 0
res['sinks_count'] = 0
res['channels_count'] = 0
flume_agent_conf_dir = flume_conf_directory + os.sep + res['name']
flume_agent_meta_file = flume_agent_conf_dir + os.sep + 'ambari-meta.json'
try:
with open(flume_agent_meta_file) as fp:
meta = json.load(fp)
res['sources_count'] = meta['sources_count']
res['sinks_count'] = meta['sinks_count']
res['channels_count'] = meta['channels_count']
except:
Logger.logger.exception(format("Error reading {flume_agent_meta_file}: "))
return res
def _check_datanode_shutdown(hdfs_binary):
"""
Checks that a DataNode is down by running "hdfs dfsamin getDatanodeInfo"
several times, pausing in between runs. Once the DataNode stops responding
this method will return, otherwise it will raise a Fail(...) and retry
automatically.
The stack defaults for retrying for HDFS are also way too slow for this
command; they are set to wait about 45 seconds between client retries. As
a result, a single execution of dfsadmin will take 45 seconds to retry and
the DataNode may be marked as dead, causing problems with HBase.
https://issues.apache.org/jira/browse/HDFS-8510 tracks reducing the
times for ipc.client.connect.retry.interval. In the meantime, override them
here, but only for RU.
:param hdfs_binary: name/path of the HDFS binary to use
:return:
"""
import params
# override stock retry timeouts since after 30 seconds, the datanode is
# marked as dead and can affect HBase during RU
dfsadmin_base_command = get_dfsadmin_base_command(hdfs_binary)
command = format('{dfsadmin_base_command} -D ipc.client.connect.max.retries=5 -D ipc.client.connect.retry.interval=1000 -getDatanodeInfo {dfs_dn_ipc_address}')
try:
Execute(command, user=params.hdfs_user, tries=1)
except:
Logger.info("DataNode has successfully shutdown for upgrade.")
return
Logger.info("DataNode has not shutdown.")
raise Fail('DataNode has not shutdown.')
def start(self, env):
import params
env.set_params(params)
self.configure(env)
start_cmd = format("service sidewinder start")
Execute(start_cmd)
print 'Start Sidewinder'
def service_check(self, env):
import params
env.set_params(params)
if params.security_enabled:
Execute(format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}"), user=params.smokeuser)
Execute("sqoop version", user=params.smokeuser, path=params.sqoop_bin_dir, logoutput=True)
def service_check(self, env):
import params
env.set_params(params)
smoke_cmd = os.path.join(params.hdp_root, "Run-SmokeTests.cmd")
service = "SQOOP"
Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True)
def pre_rolling_upgrade_shutdown(hdfs_binary):
"""
Runs the "shutdownDatanode {ipc_address} upgrade" command to shutdown the
DataNode in preparation for an upgrade. This will then periodically check
"getDatanodeInfo" to ensure the DataNode has shutdown correctly.
This function will obtain the Kerberos ticket if security is enabled.
:param hdfs_binary: name/path of the HDFS binary to use
:return: Return True if ran ok (even with errors), and False if need to stop the datanode forcefully.
"""
import params
Logger.info('DataNode executing "shutdownDatanode" command in preparation for upgrade...')
if params.security_enabled:
Execute(params.dn_kinit_cmd, user = params.hdfs_user)
dfsadmin_base_command = get_dfsadmin_base_command(hdfs_binary)
command = format('{dfsadmin_base_command} -shutdownDatanode {dfs_dn_ipc_address} upgrade')
code, output = shell.call(command, user=params.hdfs_user)
if code == 0:
# verify that the datanode is down
_check_datanode_shutdown(hdfs_binary)
else:
# Due to bug HDFS-7533, DataNode may not always shutdown during stack upgrade, and it is necessary to kill it.
if output is not None and re.search("Shutdown already in progress", output):
Logger.error("Due to a known issue in DataNode, the command {0} did not work, so will need to shutdown the datanode forcefully.".format(command))
return False
return True
def pre_upgrade_deregister():
"""
Runs the "hive --service hiveserver2 --deregister <version>" command to
de-provision the server in preparation for an upgrade. This will contact
ZooKeeper to remove the server so that clients that attempt to connect
will be directed to other servers automatically. Once all
clients have drained, the server will shutdown automatically; this process
could take a very long time.
This function will obtain the Kerberos ticket if security is enabled.
:return:
"""
import params
Logger.info('HiveServer2 executing "deregister" command in preparation for upgrade...')
if params.security_enabled:
kinit_command=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}; ")
Execute(kinit_command,user=params.smokeuser)
# calculate the current hive server version
current_hiveserver_version = _get_current_hiveserver_version()
if current_hiveserver_version is None:
raise Fail('Unable to determine the current HiveServer2 version to deregister.')
# deregister
command = 'hive --service hiveserver2 --deregister ' + current_hiveserver_version
Execute(command, user=params.hive_user, path=params.execute_path, tries=1 )
def _assert_valid(self):
source = self.main_resource.resource.source
type = self.main_resource.resource.type
target = self.main_resource.resource.target
if source:
if not os.path.exists(source):
raise Fail(format("Source {source} doesn't exist"))
if type == "directory" and os.path.isfile(source):
raise Fail(format("Source {source} is file but type is {type}"))
elif type == "file" and os.path.isdir(source):
raise Fail(format("Source {source} is directory but type is {type}"))
self.target_status = self._get_file_status(target)
if self.target_status and self.target_status['type'].lower() != type:
raise Fail(format("Trying to create file/directory but directory/file exists in the DFS on {target}"))
def install(self, env):
import params
env.set_params(params)
print 'Install the Sidewinder'
install_cmd = format("wget -O /tmp/sidewinder.rpm http://search.maven.org/remotecontent?filepath=com/srotya/sidewinder/sidewinder-cluster-dist/${project.version}/sidewinder-cluster-dist-${project.version}.rpm; rpm -ivf /tmp/sidewinder.rpm")
# stop_cmd = format("rpm -ivf /tmp/sidewinder.rpm")
Execute(install_cmd)
self.install_packages(env)
def create_topology_script():
import params
# installing the topology script to the specified location
File(params.net_topology_script_file_path,
content=StaticFile('topology_script.py'),
mode=0755,
only_if=format("test -d {net_topology_script_dir}"),
)
def service_check(self, env):
import params
env.set_params(params)
path_to_tez_jar = format(params.tez_examples_jar)
wordcount_command = format("jar {path_to_tez_jar} orderedwordcount /tmp/tezsmokeinput/sample-tez-test /tmp/tezsmokeoutput/")
test_command = format("fs -test -e /tmp/tezsmokeoutput/_SUCCESS")
File(format("{tmp_dir}/sample-tez-test"),
content = "foo\nbar\nfoo\nbar\nfoo",
mode = 0755
)
params.HdfsResource("/tmp/tezsmokeinput",
action = "create_on_execute",
type = "directory",
owner = params.smokeuser,
)
params.HdfsResource("/tmp/tezsmokeinput/sample-tez-test",
action = "create_on_execute",
type = "file",
owner = params.smokeuser,
source = format("{tmp_dir}/sample-tez-test"),
)
if params.hdp_stack_version and compare_versions(params.hdp_stack_version, '2.2.0.0') >= 0:
copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
params.HdfsResource(None, action = "execute")
ExecuteHadoop(wordcount_command,
tries = 3,
try_sleep = 5,
user = params.smokeuser,
conf_dir = params.hadoop_conf_dir,
bin_dir = params.hadoop_bin_dir
)
ExecuteHadoop(test_command,
tries = 10,
try_sleep = 6,
user = params.smokeuser,
conf_dir = params.hadoop_conf_dir,
bin_dir = params.hadoop_bin_dir
)
from resource_management.libraries.script.script import Script
# a map of the Ambari role to the component name
# for use with <stack-root>/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
'ACCUMULO_MASTER': 'accumulo-master',
'ACCUMULO_MONITOR': 'accumulo-monitor',
'ACCUMULO_GC': 'accumulo-gc',
'ACCUMULO_TRACER': 'accumulo-tracer',
'ACCUMULO_TSERVER': 'accumulo-tablet',
'ACCUMULO_CLIENT': 'accumulo-client'
}
component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP,
"ACCUMULO_CLIENT")
config = Script.get_config()
stack_root = Script.get_stack_root()
conf_dir = format('{stack_root}/current/{component_directory}/conf')
server_conf_dir = format('{conf_dir}/server')
pid_dir = config['configurations']['accumulo-env']['accumulo_pid_dir']
accumulo_user = config['configurations']['accumulo-env']['accumulo_user']
# Security related/required params
hostname = config['hostname']
security_enabled = config['configurations']['cluster-env']['security_enabled']
kinit_path_local = get_kinit_path(
default('/configurations/kerberos-env/executable_search_paths', None))
tmp_dir = Script.get_tmp_dir()
_atlas_principal_name = config['configurations']['application-properties'][
'atlas.authentication.principal']
atlas_jaas_principal = _atlas_principal_name.replace(
'_HOST', _hostname_lowercase)
atlas_keytab_path = config['configurations']['application-properties'][
'atlas.authentication.keytab']
# New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade
version = default("/commandParams/version", None)
# stack version
stack_version_unformatted = config['hostLevelParams']['stack_version']
stack_version_formatted = format_stack_version(stack_version_unformatted)
metadata_home = os.environ[
'METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else format(
'{stack_root}/current/atlas-server')
metadata_bin = format("{metadata_home}/bin")
python_binary = os.environ[
'PYTHON_EXE'] if 'PYTHON_EXE' in os.environ else sys.executable
metadata_start_script = format("{metadata_bin}/atlas_start.py")
metadata_stop_script = format("{metadata_bin}/atlas_stop.py")
# metadata local directory structure
log_dir = config['configurations']['atlas-env']['metadata_log_dir']
# service locations
hadoop_conf_dir = os.path.join(
os.environ["HADOOP_HOME"],
"conf") if 'HADOOP_HOME' in os.environ else '/etc/hadoop/conf'
ranger_plugin_config['policy.download.auth.users'] = kafka_user
ranger_plugin_config['tag.download.auth.users'] = kafka_user
ranger_plugin_config['ambari.service.check.user'] = policy_user
#For curl command in ranger plugin to get db connector
jdk_location = config['hostLevelParams']['jdk_location']
java_share_dir = '/usr/share/java'
previous_jdbc_jar_name = None
if stack_supports_ranger_audit_db:
if xa_audit_db_flavor and xa_audit_db_flavor == 'mysql':
jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name",
None)
previous_jdbc_jar_name = default(
"/hostLevelParams/previous_custom_mysql_jdbc_name", None)
audit_jdbc_url = format(
'jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
jdbc_driver = "com.mysql.jdbc.Driver"
elif xa_audit_db_flavor and xa_audit_db_flavor == 'oracle':
jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name",
None)
previous_jdbc_jar_name = default(
"/hostLevelParams/previous_custom_oracle_jdbc_name", None)
colon_count = xa_db_host.count(':')
if colon_count == 2 or colon_count == 0:
audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}')
else:
audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}')
jdbc_driver = "oracle.jdbc.OracleDriver"
elif xa_audit_db_flavor and xa_audit_db_flavor == 'postgres':
jdbc_jar_name = default(
"/hostLevelParams/custom_postgres_jdbc_name", None)
def stop(self, env):
import params
env.set_params(params)
stop_cmd = format("service sidewinder stop")
Execute(stop_cmd)
print 'Stop Sidewinder'
def get_daemon_cmd(params=None, node_type=None, command=None):
return format(
'source {params.druid_conf_dir}/druid-env.sh ; {params.druid_home}/bin/node.sh {node_type} {command}'
)
# hadoop default parameters
mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec")
hadoop_bin = stack_select.get_hadoop_dir("sbin")
hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
hadoop_home = stack_select.get_hadoop_dir("home")
hadoop_secure_dn_user = hdfs_user
hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
hadoop_conf_secure_dir = os.path.join(hadoop_conf_dir, "secure")
hadoop_lib_home = stack_select.get_hadoop_dir("lib")
# hadoop parameters for stacks that support rolling_upgrade
if stack_version_formatted and check_stack_feature(
StackFeature.ROLLING_UPGRADE, stack_version_formatted):
mapreduce_libs_path = format(
"{stack_root}/current/hadoop-mapreduce-client/*")
if not security_enabled:
hadoop_secure_dn_user = '******'
else:
dfs_dn_port = utils.get_port(dfs_dn_addr)
dfs_dn_http_port = utils.get_port(dfs_dn_http_addr)
dfs_dn_https_port = utils.get_port(dfs_dn_https_addr)
# We try to avoid inability to start datanode as a plain user due to usage of root-owned ports
if dfs_http_policy == "HTTPS_ONLY":
secure_dn_ports_are_in_use = utils.is_secure_port(
dfs_dn_port) or utils.is_secure_port(dfs_dn_https_port)
elif dfs_http_policy == "HTTP_AND_HTTPS":
secure_dn_ports_are_in_use = utils.is_secure_port(
dfs_dn_port) or utils.is_secure_port(
dfs_dn_http_port) or utils.is_secure_port(
def spark_service(name, upgrade_type=None, action=None):
import params
if action == 'start':
effective_version = params.version if upgrade_type is not None else params.stack_version_formatted
if effective_version:
effective_version = format_stack_version(effective_version)
if effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version):
# copy spark-hdp-assembly.jar to hdfs
copy_to_hdfs("spark", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
# create spark history directory
params.HdfsResource(params.spark_history_dir,
type="directory",
action="create_on_execute",
owner=params.spark_user,
group=params.user_group,
mode=0777,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
if params.security_enabled:
spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
Execute(spark_kinit_cmd, user=params.spark_user)
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
if params.stack_version_formatted and check_stack_feature(StackFeature.TEZ_FOR_SPARK, params.stack_version_formatted):
resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
if resource_created:
params.HdfsResource(None, action="execute")
if name == 'jobhistoryserver':
historyserver_no_op_test = format(
'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1')
try:
Execute(format('{spark_history_server_start}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home},
not_if=historyserver_no_op_test)
except:
show_logs(params.spark_log_dir, user=params.spark_user)
raise
elif name == 'sparkthriftserver':
if params.security_enabled:
hive_principal = params.hive_kerberos_principal.replace('_HOST', socket.getfqdn().lower())
hive_kinit_cmd = format("{kinit_path_local} -kt {hive_kerberos_keytab} {hive_principal}; ")
Execute(hive_kinit_cmd, user=params.hive_user)
thriftserver_no_op_test = format(
'ls {spark_thrift_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_thrift_server_pid_file}` >/dev/null 2>&1')
try:
Execute(format('{spark_thrift_server_start} --properties-file {spark_thrift_server_conf_file} {spark_thrift_cmd_opts_properties}'),
user=params.hive_user,
environment={'JAVA_HOME': params.java_home},
not_if=thriftserver_no_op_test
)
except:
show_logs(params.spark_log_dir, user=params.hive_user)
raise
elif action == 'stop':
if name == 'jobhistoryserver':
try:
Execute(format('{spark_history_server_stop}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home}
)
except:
show_logs(params.spark_log_dir, user=params.spark_user)
raise
File(params.spark_history_server_pid_file,
action="delete"
)
elif name == 'sparkthriftserver':
try:
Execute(format('{spark_thrift_server_stop}'),
user=params.hive_user,
environment={'JAVA_HOME': params.java_home}
)
except:
show_logs(params.spark_log_dir, user=params.hive_user)
raise
File(params.spark_thrift_server_pid_file,
action="delete"
)
def _verify_llap_app_status(self, llap_app_info, llap_app_name,
return_immediately_if_stopped, curr_time):
if llap_app_info is None or 'state' not in llap_app_info:
Logger.error(
"Malformed JSON data received for LLAP app. Exiting ....")
return False
# counters based on various states.
live_instances = 0
desired_instances = 0
percent_desired_instances_to_be_up = 80 # Used in 'RUNNING_PARTIAL' state.
if return_immediately_if_stopped and (llap_app_info['state'].upper()
in ('APP_NOT_FOUND',
'COMPLETE')):
return False
if llap_app_info['state'].upper() == 'RUNNING_ALL':
Logger.info("LLAP app '{0}' in '{1}' state.".format(
llap_app_name, llap_app_info['state']))
return True
elif llap_app_info['state'].upper() == 'RUNNING_PARTIAL':
# Check how many instances were up.
if 'liveInstances' in llap_app_info and 'desiredInstances' in llap_app_info:
live_instances = llap_app_info['liveInstances']
desired_instances = llap_app_info['desiredInstances']
else:
Logger.info(
"LLAP app '{0}' is in '{1}' state, but 'instances' information not available in JSON received. " \
"Exiting ....".format(llap_app_name, llap_app_info['state']))
Logger.info(llap_app_info)
return False
if desired_instances == 0:
Logger.info(
"LLAP app '{0}' desired instance are set to 0. Exiting ...."
.format(llap_app_name))
return False
percentInstancesUp = 0
if live_instances > 0:
percentInstancesUp = float(
live_instances) / desired_instances * 100
if percentInstancesUp >= percent_desired_instances_to_be_up:
Logger.info("LLAP app '{0}' in '{1}' state. Live Instances : '{2}' >= {3}% of Desired Instances : " \
"'{4}'.".format(llap_app_name, llap_app_info['state'],
llap_app_info['liveInstances'],
percent_desired_instances_to_be_up,
llap_app_info['desiredInstances']))
return True
else:
Logger.info("LLAP app '{0}' in '{1}' state. Live Instances : '{2}'. Desired Instances : " \
"'{3}' after {4} secs.".format(llap_app_name, llap_app_info['state'],
llap_app_info['liveInstances'],
llap_app_info['desiredInstances'],
time.time() - curr_time))
raise Fail(
"App state is RUNNING_PARTIAL. Live Instances : '{0}', Desired Instance : '{1}'"
.format(llap_app_info['liveInstances'],
llap_app_info['desiredInstances']))
elif llap_app_info['state'].upper() in [
'APP_NOT_FOUND', 'LAUNCHING', 'COMPLETE'
]:
status_str = format("LLAP app '{0}' current state is {1}.".format(
llap_app_name, llap_app_info['state']))
Logger.info(status_str)
raise Fail(status_str)
else: # Covers any unknown that we get.
Logger.info(
"LLAP app '{0}' current state is '{1}'. Expected : 'RUNNING'.".
format(llap_app_name, llap_app_info['state']))
return False
java_share_dir = '/usr/share/java'
if has_ranger_admin:
enable_ranger_nifi = (
config['configurations']['ranger-nifi-plugin-properties']['ranger-nifi-plugin-enabled'].lower() == 'yes')
xa_audit_db_password = unicode(
config['configurations']['admin-properties']['audit_db_password']) if stack_supports_ranger_audit_db else None
repo_config_password = unicode(config['configurations']['ranger-env']['admin_password'])
xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
previous_jdbc_jar_name = None
if stack_supports_ranger_audit_db:
if xa_audit_db_flavor == 'mysql':
jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None)
previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None)
audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
jdbc_driver = "com.mysql.jdbc.Driver"
elif xa_audit_db_flavor == 'oracle':
jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None)
previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None)
colon_count = xa_db_host.count(':')
if colon_count == 2 or colon_count == 0:
audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}')
else:
audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}')
jdbc_driver = "oracle.jdbc.OracleDriver"
elif xa_audit_db_flavor == 'postgres':
jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None)
previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None)
audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}')
jdbc_driver = "org.postgresql.Driver"
SERVER_ROLE_DIRECTORY_MAP, "HIVE_SERVER_INTERACTIVE")
config = Script.get_config()
stack_root = Script.get_stack_root()
stack_version_unformatted = config['hostLevelParams']['stack_version']
stack_version_formatted_major = format_stack_version(stack_version_unformatted)
if OSCheck.is_windows_family():
hive_metastore_win_service_name = "metastore"
hive_client_win_service_name = "hwi"
hive_server_win_service_name = "hiveserver2"
webhcat_server_win_service_name = "templeton"
else:
hive_pid_dir = config['configurations']['hive-env']['hive_pid_dir']
hive_pid = format("{hive_pid_dir}/hive-server.pid")
hive_interactive_pid = format("{hive_pid_dir}/hive-interactive.pid")
hive_metastore_pid = format("{hive_pid_dir}/hive.pid")
hcat_pid_dir = config['configurations']['hive-env'][
'hcat_pid_dir'] #hcat_pid_dir
webhcat_pid_file = format('{hcat_pid_dir}/webhcat.pid')
process_name = 'mysqld'
if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
daemon_name = 'mysql'
else:
daemon_name = 'mysqld'
# Security related/required params
hostname = config['hostname']
hadoop_lib_home = stack_select.get_hadoop_dir("lib")
#hadoop params
if stack_version_formatted and check_stack_feature(
StackFeature.ROLLING_UPGRADE, stack_version_formatted):
stack_version = None
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is not None and len(
upgrade_stack) == 2 and upgrade_stack[1] is not None:
stack_version = upgrade_stack[1]
# oozie-server or oozie-client, depending on role
oozie_root = status_params.component_directory
# using the correct oozie root dir, format the correct location
oozie_lib_dir = format("{stack_root}/current/{oozie_root}")
oozie_setup_sh = format(
"{stack_root}/current/{oozie_root}/bin/oozie-setup.sh")
oozie_webapps_dir = format(
"{stack_root}/current/{oozie_root}/oozie-server/webapps")
oozie_webapps_conf_dir = format(
"{stack_root}/current/{oozie_root}/oozie-server/conf")
oozie_libext_dir = format("{stack_root}/current/{oozie_root}/libext")
oozie_server_dir = format("{stack_root}/current/{oozie_root}/oozie-server")
oozie_shared_lib = format("{stack_root}/current/{oozie_root}/share")
oozie_home = format("{stack_root}/current/{oozie_root}")
oozie_bin_dir = format("{stack_root}/current/{oozie_root}/bin")
oozie_examples_regex = format("{stack_root}/current/{oozie_root}/doc")
# set the falcon home for copying JARs; if in an upgrade, then use the version of falcon that
# matches the version of oozie
if has_zk_host:
if 'zoo.cfg' in config['configurations'] and 'clientPort' in config[
'configurations']['zoo.cfg']:
zookeeper_clientPort = config['configurations']['zoo.cfg'][
'clientPort']
else:
zookeeper_clientPort = '2181'
zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(
config['clusterHostInfo']['zookeeper_hosts'])
# last port config
zookeeper_quorum += ':' + zookeeper_clientPort
#hadoop params
if has_namenode or dfs_type == 'HCFS':
hadoop_tmp_dir = format("/tmp/hadoop-{hdfs_user}")
hadoop_conf_dir = conf_select.get_hadoop_conf_dir(
force_latest_on_upgrade=True)
task_log4j_properties_location = os.path.join(hadoop_conf_dir,
"task-log4j.properties")
hadoop_pid_dir_prefix = config['configurations']['hadoop-env'][
'hadoop_pid_dir_prefix']
hdfs_log_dir_prefix = config['configurations']['hadoop-env'][
'hdfs_log_dir_prefix']
hbase_tmp_dir = "/tmp/hbase-hbase"
#db params
server_db_name = config['hostLevelParams']['db_name']
db_driver_filename = config['hostLevelParams']['db_driver_filename']
oracle_driver_url = config['hostLevelParams']['oracle_jdbc_url']
mysql_driver_url = config['hostLevelParams']['mysql_jdbc_url']
stack_supports_ranger_kerberos = stack_version_formatted and check_stack_feature(
StackFeature.RANGER_KERBEROS_SUPPORT, stack_version_formatted)
# hadoop default parameters
hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
daemon_script = "/usr/lib/hbase/bin/hbase-daemon.sh"
region_mover = "/usr/lib/hbase/bin/region_mover.rb"
region_drainer = "/usr/lib/hbase/bin/draining_servers.rb"
hbase_cmd = "/usr/lib/hbase/bin/hbase"
hbase_max_direct_memory_size = None
# hadoop parameters for stacks supporting rolling_upgrade
if stack_version_formatted and check_stack_feature(
StackFeature.ROLLING_UPGRADE, stack_version_formatted):
daemon_script = format(
'{stack_root}/current/hbase-client/bin/hbase-daemon.sh')
region_mover = format(
'{stack_root}/current/hbase-client/bin/region_mover.rb')
region_drainer = format(
'{stack_root}/current/hbase-client/bin/draining_servers.rb')
hbase_cmd = format('{stack_root}/current/hbase-client/bin/hbase')
hbase_max_direct_memory_size = default(
'configurations/hbase-env/hbase_max_direct_memory_size', None)
daemon_script = format(
"{stack_root}/current/{component_directory}/bin/hbase-daemon.sh")
region_mover = format(
"{stack_root}/current/{component_directory}/bin/region_mover.rb")
region_drainer = format(
"{stack_root}/current/{component_directory}/bin/draining_servers.rb")
def _llap_start(self, env, cleanup=False):
import params
env.set_params(params)
if params.hive_server_interactive_ha:
"""
Check llap app state
"""
Logger.info(
"HSI HA is enabled. Checking if LLAP is already running ...")
if params.stack_supports_hive_interactive_ga:
status = self.check_llap_app_status_in_llap_ga(
params.llap_app_name, 2, params.hive_server_interactive_ha)
else:
status = self.check_llap_app_status_in_llap_tp(
params.llap_app_name, 2, params.hive_server_interactive_ha)
if status:
Logger.info("LLAP app '{0}' is already running.".format(
params.llap_app_name))
return True
else:
Logger.info(
"LLAP app '{0}' is not running. llap will be started.".
format(params.llap_app_name))
pass
# Call for cleaning up the earlier run(s) LLAP package folders.
self._cleanup_past_llap_package_dirs()
Logger.info("Starting LLAP")
LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir()
unique_name = "llap-slider%s" % datetime.utcnow().strftime(
'%Y-%m-%d_%H-%M-%S')
cmd = format(
"{stack_root}/current/hive-server2-hive2/bin/hive --service llap --slider-am-container-mb {params.slider_am_container_mb} "
"--size {params.llap_daemon_container_size}m --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m "
"--loglevel {params.llap_log_level} {params.llap_extra_slider_opts} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}"
)
# Append params that are supported from Hive llap GA version.
if params.stack_supports_hive_interactive_ga:
# Figure out the Slider Anti-affinity to be used.
# YARN does not support anti-affinity, and therefore Slider implements AA by the means of exclusion lists, i.e, it
# starts containers one by one and excludes the nodes it gets (adding a delay of ~2sec./machine). When the LLAP
# container memory size configuration is more than half of YARN node memory, AA is implicit and should be avoided.
slider_placement = 4
if long(params.llap_daemon_container_size) > (
0.5 * long(params.yarn_nm_mem)):
slider_placement = 0
Logger.info(
"Setting slider_placement : 0, as llap_daemon_container_size : {0} > 0.5 * "
"YARN NodeManager Memory({1})".format(
params.llap_daemon_container_size, params.yarn_nm_mem))
else:
Logger.info(
"Setting slider_placement: 4, as llap_daemon_container_size : {0} <= 0.5 * "
"YARN NodeManager Memory({1})".format(
params.llap_daemon_container_size, params.yarn_nm_mem))
cmd += format(
" --slider-placement {slider_placement} --skiphadoopversion --skiphbasecp --instances {params.num_llap_daemon_running_nodes}"
)
# Setup the logger for the ga version only
cmd += format(" --logger {params.llap_logger}")
else:
cmd += format(" --instances {params.num_llap_nodes}")
if params.security_enabled:
llap_keytab_splits = params.hive_llap_keytab_file.split("/")
Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits))
cmd += format(
" --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab "
"{llap_keytab_splits[4]} --slider-principal {params.hive_llap_principal}"
)
# Add the aux jars if they are specified. If empty, dont need to add this param.
if params.hive_aux_jars:
cmd += format(" --auxjars {params.hive_aux_jars}")
# Append args.
llap_java_args = InlineTemplate(
params.llap_app_java_opts).get_content()
cmd += format(" --args \" {llap_java_args}\"")
# Append metaspace size to args.
if params.java_version > 7 and params.llap_daemon_container_size > 4096:
if params.llap_daemon_container_size <= 32768:
metaspaceSize = "256m"
else:
metaspaceSize = "1024m"
cmd = cmd[:-1] + " -XX:MetaspaceSize=" + metaspaceSize + "\""
run_file_path = None
try:
Logger.info(format("LLAP start command: {cmd}"))
code, output, error = shell.checked_call(cmd,
user=params.hive_user,
quiet=True,
stderr=subprocess.PIPE,
logoutput=True)
if code != 0 or output is None:
raise Fail(
"Command failed with either non-zero return code or no output."
)
# E.g., output:
# Prepared llap-slider-05Apr2016/run.sh for running LLAP on Slider
exp = r"Prepared (.*?run.sh) for running LLAP"
run_file_path = None
out_splits = output.split("\n")
for line in out_splits:
line = line.strip()
m = re.match(exp, line, re.I)
if m and len(m.groups()) == 1:
run_file_name = m.group(1)
run_file_path = os.path.join(params.hive_user_home_dir,
run_file_name)
break
if not run_file_path:
raise Fail("Did not find run.sh file in output: " +
str(output))
Logger.info(format("Run file path: {run_file_path}"))
Execute(run_file_path, user=params.hive_user, logoutput=True)
Logger.info("Submitted LLAP app name : {0}".format(
params.llap_app_name))
# We need to check the status of LLAP app to figure out it got
# launched properly and is in running state. Then go ahead with Hive Interactive Server start.
if params.stack_supports_hive_interactive_ga:
status = self.check_llap_app_status_in_llap_ga(
params.llap_app_name,
params.num_retries_for_checking_llap_status)
else:
status = self.check_llap_app_status_in_llap_tp(
params.llap_app_name,
params.num_retries_for_checking_llap_status)
if status:
Logger.info("LLAP app '{0}' deployed successfully.".format(
params.llap_app_name))
return True
else:
Logger.error("LLAP app '{0}' deployment unsuccessful.".format(
params.llap_app_name))
return False
except:
# Attempt to clean up the packaged application, or potentially rename it with a .bak
if run_file_path is not None and cleanup:
parent_dir = os.path.dirname(run_file_path)
Directory(
parent_dir,
action="delete",
ignore_failures=True,
)
# throw the original exception
raise
def status(self, env):
import params
env.set_params(params)
status_cmd = format("service sidewinder status")
Execute(status_cmd)
print 'Status of the Master'
'flume_principal_name']
flume_keytab_path = config['configurations']['flume-env'][
'flume_keytab_path']
stack_version_unformatted = config['clusterLevelParams']['stack_version']
stack_version_formatted = format_stack_version(stack_version_unformatted)
# hadoop default parameters
flume_bin = '/usr/bin/flume-ng'
flume_hive_home = '/usr/lib/hive'
flume_hcat_home = '/usr/lib/hive-hcatalog'
# hadoop parameters for stack supporting rolling upgrade
if stack_version_formatted and check_stack_feature(
StackFeature.ROLLING_UPGRADE, stack_version_formatted):
flume_bin = format('{stack_root}/current/flume-server/bin/flume-ng')
flume_hive_home = format('{stack_root}/current/hive-metastore')
flume_hcat_home = format('{stack_root}/current/hive-webhcat')
java_home = config['ambariLevelParams']['java_home']
flume_log_dir = config['configurations']['flume-env']['flume_log_dir']
flume_run_dir = config['configurations']['flume-env']['flume_run_dir']
ambari_state_file = format("{flume_run_dir}/ambari-state.txt")
if (('flume-conf' in config['configurations'])
and ('content' in config['configurations']['flume-conf'])):
flume_conf_content = config['configurations']['flume-conf']['content']
else:
flume_conf_content = None
if (('flume-log4j' in config['configurations'])
# hadoop default parameters
flume_bin = '/usr/bin/flume-ng'
flume_hive_home = '/usr/lib/hive'
flume_hcat_home = '/usr/lib/hive-hcatalog'
# hadoop parameters for 2.2+
if Script.is_stack_greater_or_equal("2.2"):
flume_bin = '/usr/hdp/current/flume-server/bin/flume-ng'
flume_hive_home = '/usr/hdp/current/hive-metastore'
flume_hcat_home = '/usr/hdp/current/hive-webhcat'
java_home = config['hostLevelParams']['java_home']
flume_log_dir = config['configurations']['flume-env']['flume_log_dir']
flume_run_dir = config['configurations']['flume-env']['flume_run_dir']
ambari_state_file = format("{flume_run_dir}/ambari-state.txt")
if (('flume-conf' in config['configurations']) and('content' in config['configurations']['flume-conf'])):
flume_conf_content = config['configurations']['flume-conf']['content']
else:
flume_conf_content = None
if (('flume-log4j' in config['configurations']) and ('content' in config['configurations']['flume-log4j'])):
flume_log4j_content = config['configurations']['flume-log4j']['content']
else:
flume_log4j_content = None
targets = default('/commandParams/flume_handler', None)
flume_command_targets = [] if targets is None else targets.split(',')
flume_env_sh_template = config['configurations']['flume-env']['content']
from resource_management.libraries.functions import default
from resource_management.libraries.functions import format
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import hdp_select
from resource_management.libraries.functions import format_jvm_option
from resource_management.libraries.functions.is_empty import is_empty
from resource_management.libraries.functions.version import format_hdp_stack_version
from resource_management.libraries.functions.version import compare_versions
from ambari_commons.os_check import OSCheck
from ambari_commons.constants import AMBARI_SUDO_BINARY
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
artifact_dir = format("{tmp_dir}/AMBARI-artifacts/")
jdk_name = default("/hostLevelParams/jdk_name", None)
java_home = config['hostLevelParams']['java_home']
java_version = int(config['hostLevelParams']['java_version'])
jdk_location = config['hostLevelParams']['jdk_location']
sudo = AMBARI_SUDO_BINARY
ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
restart_type = default("/commandParams/restart_type", "")
version = default("/commandParams/version", None)
# Handle upgrade and downgrade
stack_name = status_params.stack_name
component_directory = status_params.component_directory
# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
version = default("/commandParams/version", None)
# default parameters
zk_home = "/usr"
zk_bin = "/usr/lib/zookeeper/bin"
zk_cli_shell = "/usr/lib/zookeeper/bin/zkCli.sh"
config_dir = "/etc/zookeeper/conf"
zk_smoke_out = os.path.join(tmp_dir, "zkSmoke.out")
# hadoop parameters for stacks that support rolling_upgrade
if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
zk_home = format("{stack_root}/current/{component_directory}")
zk_bin = format("{stack_root}/current/{component_directory}/bin")
zk_cli_shell = format("{stack_root}/current/{component_directory}/bin/zkCli.sh")
config_dir = status_params.config_dir
zk_user = config['configurations']['zookeeper-env']['zk_user']
hostname = config['agentLevelParams']['hostname']
user_group = config['configurations']['cluster-env']['user_group']
zk_env_sh_template = config['configurations']['zookeeper-env']['content']
zk_log_dir = config['configurations']['zookeeper-env']['zk_log_dir']
zk_data_dir = config['configurations']['zoo.cfg']['dataDir']
zk_pid_dir = status_params.zk_pid_dir
zk_pid_file = status_params.zk_pid_file
zk_server_heapsize_value = str(default('configurations/zookeeper-env/zk_server_heapsize', "1024"))
# server configurations
config = Script.get_config()
stack_root = Script.get_stack_root()
stack_name = default("/hostLevelParams/stack_name", None)
user_group = config['configurations']['cluster-env']['user_group']
# stack version
stack_version = default("/commandParams/version", None)
hostname = config['hostname']
# status params
status_pid_dir = status_params.superset_pid_dir
superset_home_dir = format("{stack_root}/current/superset")
superset_bin_dir = format("{superset_home_dir}/bin")
superset_log_dir = default("/configurations/superset-env/superset_log_dir",
'/var/log/superset')
superset_pid_dir = status_params.superset_pid_dir
superset_config_dir = '/etc/superset/conf'
superset_admin_user = config['configurations']['superset-env'][
'superset_admin_user']
superset_admin_password = config['configurations']['superset-env'][
'superset_admin_password']
superset_admin_firstname = config['configurations']['superset-env'][
'superset_admin_firstname']
superset_admin_lastname = config['configurations']['superset-env'][
'superset_admin_lastname']
superset_admin_email = config['configurations']['superset-env'][
'superset_admin_email']
# E.g., 2.3
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
# This is the version whose state is CURRENT. During an RU, this is the source version.
# DO NOT format it since we need the build number too.
upgrade_from_version = default("/hostLevelParams/current_version", None)
# server configurations
# Default value used in HDP 2.3.0.0 and earlier.
knox_data_dir = '/var/lib/knox/data'
# Important, it has to be strictly greater than 2.3.0.0!!!
if stack_name and stack_name.upper() == "HDP":
Logger.info(format("HDP version to use is {version_formatted}"))
if Script.is_hdp_stack_greater(version_formatted, "2.3.0.0"):
# This is the current version. In the case of a Rolling Upgrade, it will be the newer version.
# In the case of a Downgrade, it will be the version downgrading to.
# This is always going to be a symlink to /var/lib/knox/data_${version}
knox_data_dir = format('/usr/hdp/{version}/knox/data')
Logger.info(
format(
"Detected HDP with stack version {version}, will use knox_data_dir = {knox_data_dir}"
))
knox_master_secret_path = format('{knox_data_dir}/security/master')
knox_cert_store_path = format('{knox_data_dir}/security/keystores/gateway.jks')
knox_user = default("/configurations/knox-env/knox_user", "knox")
# server configurations
stack_version_unformatted = config['hostLevelParams']['stack_version']
stack_version_formatted = format_stack_version(stack_version_unformatted)
has_secure_user_auth = False
if stack_version_formatted and \
check_stack_feature(StackFeature.ACCUMULO_KERBEROS_USER_AUTH, stack_version_formatted):
has_secure_user_auth = True
# configuration directories
conf_dir = status_params.conf_dir
server_conf_dir = status_params.server_conf_dir
# service locations
hadoop_prefix = stack_select.get_hadoop_dir("home")
hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
zookeeper_home = format("{stack_root}/current/zookeeper-client")
# the configuration direction for HDFS/YARN/MapR is the hadoop config
# directory, which is symlinked by hadoop-client only
hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
# accumulo local directory structure
log_dir = config['configurations']['accumulo-env']['accumulo_log_dir']
client_script = format("{stack_root}/current/accumulo-client/bin/accumulo")
daemon_script = format("ACCUMULO_CONF_DIR={server_conf_dir} {client_script}")
# user and status
accumulo_user = status_params.accumulo_user
user_group = config['configurations']['cluster-env']['user_group']
pid_dir = status_params.pid_dir
def kafka(upgrade_type=None):
import params
ensure_base_directories()
kafka_server_config = mutable_config_dict(
params.config['configurations']['kafka-broker'])
# This still has an issue of hostnames being alphabetically out-of-order for broker.id in HDP-2.2.
# Starting in HDP 2.3, Kafka handles the generation of broker.id so Ambari doesn't have to.
effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(
params.version)
Logger.info(format("Effective stack version: {effective_version}"))
if effective_version is not None and effective_version != "" and \
check_stack_feature(StackFeature.CREATE_KAFKA_BROKER_ID, effective_version):
if len(params.kafka_hosts
) > 0 and params.hostname in params.kafka_hosts:
brokerid = str(sorted(params.kafka_hosts).index(params.hostname))
kafka_server_config['broker.id'] = brokerid
Logger.info(format("Calculating broker.id as {brokerid}"))
# listeners and advertised.listeners are only added in 2.3.0.0 onwards.
if effective_version is not None and effective_version != "" and \
check_stack_feature(StackFeature.KAFKA_LISTENERS, effective_version):
listeners = kafka_server_config['listeners'].replace(
"localhost", params.hostname)
Logger.info(format("Kafka listeners: {listeners}"))
if params.security_enabled and params.kafka_kerberos_enabled:
Logger.info("Kafka kerberos security is enabled.")
if "SASL" not in listeners:
listeners = listeners.replace("PLAINTEXT", "PLAINTEXTSASL")
kafka_server_config['listeners'] = listeners
kafka_server_config['advertised.listeners'] = listeners
Logger.info(format("Kafka advertised listeners: {listeners}"))
else:
kafka_server_config['listeners'] = listeners
if 'advertised.listeners' in kafka_server_config:
advertised_listeners = kafka_server_config[
'advertised.listeners'].replace("localhost",
params.hostname)
kafka_server_config[
'advertised.listeners'] = advertised_listeners
Logger.info(
format(
"Kafka advertised listeners: {advertised_listeners}"))
else:
kafka_server_config['host.name'] = params.hostname
if params.has_metric_collector:
kafka_server_config[
'kafka.timeline.metrics.host'] = params.metric_collector_host
kafka_server_config[
'kafka.timeline.metrics.port'] = params.metric_collector_port
kafka_server_config[
'kafka.timeline.metrics.protocol'] = params.metric_collector_protocol
kafka_server_config[
'kafka.timeline.metrics.truststore.path'] = params.metric_truststore_path
kafka_server_config[
'kafka.timeline.metrics.truststore.type'] = params.metric_truststore_type
kafka_server_config[
'kafka.timeline.metrics.truststore.password'] = params.metric_truststore_password
kafka_data_dir = kafka_server_config['log.dirs']
kafka_data_dirs = filter(None, kafka_data_dir.split(","))
Directory(
kafka_data_dirs,
mode=0755,
cd_access='a',
owner=params.kafka_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True,
)
PropertiesFile(
"server.properties",
dir=params.conf_dir,
properties=kafka_server_config,
owner=params.kafka_user,
group=params.user_group,
)
File(format("{conf_dir}/kafka-env.sh"),
owner=params.kafka_user,
content=InlineTemplate(params.kafka_env_sh_template))
if (params.log4j_props != None):
File(format("{conf_dir}/log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.kafka_user,
content=params.log4j_props)
if params.security_enabled and params.kafka_kerberos_enabled:
TemplateConfig(format("{conf_dir}/kafka_jaas.conf"),
owner=params.kafka_user)
TemplateConfig(format("{conf_dir}/kafka_client_jaas.conf"),
owner=params.kafka_user)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'kafka.conf'),
owner='root',
group='root',
mode=0644,
content=Template("kafka.conf.j2"))
File(os.path.join(params.conf_dir, 'tools-log4j.properties'),
owner='root',
group='root',
mode=0644,
content=Template("tools-log4j.properties.j2"))
setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir)
setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)
stack_supports_ranger_audit_db = check_stack_feature(
StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks)
# hadoop default parameters
hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
daemon_script = "/usr/lib/hbase/bin/hbase-daemon.sh"
region_mover = "/usr/lib/hbase/bin/region_mover.rb"
region_drainer = "/usr/lib/hbase/bin/draining_servers.rb"
hbase_cmd = "/usr/lib/hbase/bin/hbase"
hbase_max_direct_memory_size = None
# hadoop parameters for stacks supporting rolling_upgrade
if stack_version_formatted and check_stack_feature(
StackFeature.ROLLING_UPGRADE, stack_version_formatted):
daemon_script = format(
'{stack_root}/current/hbase-client/bin/hbase-daemon.sh')
region_mover = format(
'{stack_root}/current/hbase-client/bin/region_mover.rb')
region_drainer = format(
'{stack_root}/current/hbase-client/bin/draining_servers.rb')
hbase_cmd = format('{stack_root}/current/hbase-client/bin/hbase')
hbase_max_direct_memory_size = default(
'configurations/hbase-env/hbase_max_direct_memory_size', None)
daemon_script = format(
"{stack_root}/current/{component_directory}/bin/hbase-daemon.sh")
region_mover = format(
"{stack_root}/current/{component_directory}/bin/region_mover.rb")
region_drainer = format(
"{stack_root}/current/{component_directory}/bin/draining_servers.rb")
if has_kafka_host:
if 'port' in config['configurations']['kafka-broker']:
kafka_broker_port = config['configurations']['kafka-broker']['port']
else:
kafka_broker_port = '6667'
kafka_brokers = (':' + kafka_broker_port + ',').join(
config['clusterHostInfo']['kafka_broker_hosts'])
kafka_brokers += ':' + kafka_broker_port
metron_apps_hdfs_dir = config['configurations']['metron-env'][
'metron_apps_hdfs_dir']
# the double "format" is not an error - we are pulling in a jinja-templated param. This is a bit of a hack, but works
# well enough until we find a better way via Ambari
metron_apps_indexed_hdfs_dir = format(
format(config['configurations']['metron-env']
['metron_apps_indexed_hdfs_dir']))
metron_topic_retention = config['configurations']['metron-env'][
'metron_topic_retention']
local_grok_patterns_dir = format("{metron_home}/patterns")
hdfs_grok_patterns_dir = format("{metron_apps_hdfs_dir}/patterns")
# for create_hdfs_directory
security_enabled = config['configurations']['cluster-env']['security_enabled']
hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
hdfs_principal_name = config['configurations']['hadoop-env'][
'hdfs_principal_name']
smokeuser_principal = config['configurations']['cluster-env'][
'smokeuser_principal_name']
def druid(upgrade_type=None, nodeType=None):
import params
ensure_base_directories()
# Environment Variables
File(format("{params.druid_conf_dir}/druid-env.sh"),
owner=params.druid_user,
content=InlineTemplate(params.druid_env_sh_template))
# common config
druid_common_config = mutable_config_dict(
params.config['configurations']['druid-common'])
# User cannot override below configs
druid_common_config['druid.host'] = params.hostname
druid_common_config[
'druid.extensions.directory'] = params.druid_extensions_dir
druid_common_config[
'druid.extensions.hadoopDependenciesDir'] = params.druid_hadoop_dependencies_dir
druid_common_config[
'druid.selectors.indexing.serviceName'] = params.config[
'configurations']['druid-overlord']['druid.service']
druid_common_config['druid.selectors.coordinator.serviceName'] = \
params.config['configurations']['druid-coordinator']['druid.service']
# delete the password and user if empty otherwiswe derby will fail.
if 'derby' == druid_common_config['druid.metadata.storage.type']:
del druid_common_config['druid.metadata.storage.connector.user']
del druid_common_config['druid.metadata.storage.connector.password']
druid_env_config = mutable_config_dict(
params.config['configurations']['druid-env'])
PropertiesFile(
"common.runtime.properties",
dir=params.druid_common_conf_dir,
properties=druid_common_config,
owner=params.druid_user,
group=params.user_group,
)
Logger.info("Created common.runtime.properties")
File(format("{params.druid_common_conf_dir}/druid-log4j.xml"),
mode=0644,
owner=params.druid_user,
group=params.user_group,
content=InlineTemplate(params.log4j_props))
Logger.info("Created log4j file")
File("/etc/logrotate.d/druid",
mode=0644,
owner='root',
group='root',
content=InlineTemplate(params.logrotate_props))
Logger.info("Created log rotate file")
# Write Hadoop Configs if configured
if 'core-site' in params.config['configurations']:
XmlConfig(
"core-site.xml",
conf_dir=params.druid_common_conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configuration_attributes']
['core-site'],
owner=params.druid_user,
group=params.user_group)
if 'mapred-site' in params.config['configurations']:
XmlConfig(
"mapred-site.xml",
conf_dir=params.druid_common_conf_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configuration_attributes']
['mapred-site'],
owner=params.druid_user,
group=params.user_group)
if 'yarn-site' in params.config['configurations']:
XmlConfig(
"yarn-site.xml",
conf_dir=params.druid_common_conf_dir,
configurations=params.config['configurations']['yarn-site'],
configuration_attributes=params.config['configuration_attributes']
['yarn-site'],
owner=params.druid_user,
group=params.user_group)
if 'hdfs-site' in params.config['configurations']:
XmlConfig(
"hdfs-site.xml",
conf_dir=params.druid_common_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configuration_attributes']
['hdfs-site'],
owner=params.druid_user,
group=params.user_group)
# node specific configs
for node_type in [
'coordinator', 'overlord', 'historical', 'broker', 'middleManager',
'router'
]:
node_config_dir = format('{params.druid_conf_dir}/{node_type}')
node_type_lowercase = node_type.lower()
# Write runtime.properties file
node_config = mutable_config_dict(params.config['configurations'][
format('druid-{node_type_lowercase}')])
PropertiesFile(
"runtime.properties",
dir=node_config_dir,
properties=node_config,
owner=params.druid_user,
group=params.user_group,
)
Logger.info(
format("Created druid-{node_type_lowercase} runtime.properties"))
# Write jvm configs
File(
format('{node_config_dir}/jvm.config'),
owner=params.druid_user,
group=params.user_group,
content=InlineTemplate(
"-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
node_heap_memory=druid_env_config[format(
'druid.{node_type_lowercase}.jvm.heap.memory')],
log4j_config_file=format(
"{params.druid_common_conf_dir}/druid-log4j.xml"),
node_direct_memory=druid_env_config[format(
'druid.{node_type_lowercase}.jvm.direct.memory')],
node_jvm_opts=druid_env_config[format(
'druid.{node_type_lowercase}.jvm.opts')]))
Logger.info(format("Created druid-{node_type_lowercase} jvm.config"))
# All druid nodes have dependency on hdfs_client
ensure_hadoop_directories()
download_database_connector_if_needed()
# Pull all required dependencies
pulldeps()
def recommendDruidConfigurations(self, configurations, clusterData,
services, hosts):
# druid is not in list of services to be installed
if 'druid-common' not in services['configurations']:
return
componentsListList = [
service["components"] for service in services["services"]
]
componentsList = [
item["StackServiceComponents"] for sublist in componentsListList
for item in sublist
]
servicesList = [
service["StackServices"]["service_name"]
for service in services["services"]
]
putCommonProperty = self.putProperty(configurations, "druid-common",
services)
putCommonProperty('druid.zk.service.host',
self.getZKHostPortString(services))
self.recommendDruidMaxMemoryLimitConfigurations(
configurations, clusterData, services, hosts)
# recommending the metadata storage uri
database_name = services['configurations']["druid-common"][
"properties"]["database_name"]
metastore_hostname = services['configurations']["druid-common"][
"properties"]["metastore_hostname"]
database_type = services['configurations']["druid-common"][
"properties"]["druid.metadata.storage.type"]
metadata_storage_port = "1527"
mysql_module_name = "mysql-metadata-storage"
postgres_module_name = "postgresql-metadata-storage"
extensions_load_list = services['configurations']['druid-common'][
'properties']['druid.extensions.loadList']
putDruidCommonProperty = self.putProperty(configurations,
"druid-common", services)
extensions_load_list = self.removeFromList(extensions_load_list,
mysql_module_name)
extensions_load_list = self.removeFromList(extensions_load_list,
postgres_module_name)
if database_type == 'mysql':
metadata_storage_port = "3306"
extensions_load_list = self.addToList(extensions_load_list,
mysql_module_name)
if database_type == 'postgresql':
extensions_load_list = self.addToList(extensions_load_list,
postgres_module_name)
metadata_storage_port = "5432"
putDruidCommonProperty('druid.metadata.storage.connector.port',
metadata_storage_port)
putDruidCommonProperty(
'druid.metadata.storage.connector.connectURI',
self.getMetadataConnectionString(database_type).format(
metastore_hostname, database_name, metadata_storage_port))
# HDFS is installed
if "HDFS" in servicesList and "hdfs-site" in services["configurations"]:
# recommend HDFS as default deep storage
extensions_load_list = self.addToList(extensions_load_list,
"druid-hdfs-storage")
putCommonProperty("druid.storage.type", "hdfs")
putCommonProperty("druid.storage.storageDirectory",
"/user/druid/data")
# configure indexer logs configs
putCommonProperty("druid.indexer.logs.type", "hdfs")
putCommonProperty("druid.indexer.logs.directory",
"/user/druid/logs")
if "KAFKA" in servicesList:
extensions_load_list = self.addToList(
extensions_load_list, "druid-kafka-indexing-service")
if 'AMBARI_METRICS' in servicesList:
extensions_load_list = self.addToList(extensions_load_list,
"ambari-metrics-emitter")
putCommonProperty('druid.extensions.loadList', extensions_load_list)
# JVM Configs go to env properties
putEnvProperty = self.putProperty(configurations, "druid-env",
services)
# processing thread pool Config
for component in ['DRUID_HISTORICAL', 'DRUID_BROKER']:
component_hosts = self.getHostsWithComponent(
"DRUID", component, services, hosts)
nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component]
putComponentProperty = self.putProperty(configurations,
format("druid-{nodeType}"),
services)
if (component_hosts is not None and len(component_hosts) > 0):
totalAvailableCpu = self.getMinCpu(component_hosts)
processingThreads = 1
if totalAvailableCpu > 1:
processingThreads = totalAvailableCpu - 1
putComponentProperty('druid.processing.numThreads',
processingThreads)
putComponentProperty(
'druid.server.http.numThreads',
max(10, (totalAvailableCpu * 17) / 16 + 2) + 30)
# superset is in list of services to be installed
if 'druid-superset' in services['configurations']:
# Recommendations for Superset
superset_database_type = services['configurations'][
"druid-superset"]["properties"]["SUPERSET_DATABASE_TYPE"]
putSupersetProperty = self.putProperty(configurations,
"druid-superset", services)
if superset_database_type == "mysql":
putSupersetProperty("SUPERSET_DATABASE_PORT", "3306")
elif superset_database_type == "postgresql":
putSupersetProperty("SUPERSET_DATABASE_PORT", "5432")
def getPid(params=None, nodeType=None):
return format('{params.druid_pid_dir}/{nodeType}.pid')
# hadoop default parameters
mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec")
hadoop_bin = stack_select.get_hadoop_dir("sbin")
hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
hadoop_home = stack_select.get_hadoop_dir("home")
hadoop_secure_dn_user = hdfs_user
hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
hadoop_conf_secure_dir = os.path.join(hadoop_conf_dir, "secure")
hadoop_lib_home = stack_select.get_hadoop_dir("lib")
# hadoop parameters for stacks that support rolling_upgrade
if stack_version_formatted and check_stack_feature(
StackFeature.ROLLING_UPGRADE, stack_version_formatted):
mapreduce_libs_path = format(
"{stack_root}/current/hadoop-mapreduce-client/*")
if not security_enabled:
hadoop_secure_dn_user = '******'
else:
dfs_dn_port = utils.get_port(dfs_dn_addr)
dfs_dn_http_port = utils.get_port(dfs_dn_http_addr)
dfs_dn_https_port = utils.get_port(dfs_dn_https_addr)
# We try to avoid inability to start datanode as a plain user due to usage of root-owned ports
if dfs_http_policy == "HTTPS_ONLY":
secure_dn_ports_are_in_use = utils.is_secure_port(
dfs_dn_port) or utils.is_secure_port(dfs_dn_https_port)
elif dfs_http_policy == "HTTP_AND_HTTPS":
secure_dn_ports_are_in_use = utils.is_secure_port(
dfs_dn_port) or utils.is_secure_port(
dfs_dn_http_port) or utils.is_secure_port(
def prepare_libext_directory(upgrade_type=None):
"""
Performs the following actions on libext:
- creates <stack-root>/current/oozie/libext and recursively
- set 777 permissions on it and its parents.
- downloads JDBC driver JAR if needed
- copies Falcon JAR for the Oozie WAR if needed
"""
import params
# some stack versions don't need the lzo compression libraries
target_version_needs_compression_libraries = params.version and check_stack_feature(
StackFeature.LZO, params.version)
# ensure the directory exists
Directory(params.oozie_libext_dir, mode=0777)
# get all hadooplzo* JAR files
# <stack-selector-tool> set hadoop-client has not run yet, therefore we cannot use
# <stack-root>/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 2.2.0.0; don't do this
# for downgrade to 2.2.0.0 since hadoop-lzo will not be present
# This can also be called during a Downgrade.
# When a version is Installed, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
if params.lzo_enabled and (
params.upgrade_direction == Direction.UPGRADE
or target_version_needs_compression_libraries):
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format(
"{stack_root}/{version}/hadoop/lib")
files = glob.iglob(
os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
if not files:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy files into libext
files_copied = False
for file in files:
if os.path.isfile(file):
Logger.info("Copying {0} to {1}".format(
str(file), params.oozie_libext_dir))
shutil.copy2(file, params.oozie_libext_dir)
files_copied = True
if not files_copied:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# something like <stack-root>/current/oozie-server/libext/ext-2.2.zip
oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir,
params.ext_js_file)
# Copy ext ZIP to libext dir
# Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
source_ext_zip_paths = oozie.get_oozie_ext_zip_source_paths(
upgrade_type, params)
found_at_least_one_oozie_ext_file = False
# Copy the first oozie ext-2.2.zip file that is found.
# This uses a list to handle the cases when migrating from some versions of BigInsights to HDP.
if source_ext_zip_paths is not None:
for source_ext_zip_path in source_ext_zip_paths:
if os.path.isfile(source_ext_zip_path):
found_at_least_one_oozie_ext_file = True
Logger.info("Copying {0} to {1}".format(
source_ext_zip_path, params.oozie_libext_dir))
Execute(
("cp", source_ext_zip_path, params.oozie_libext_dir),
sudo=True)
Execute(("chown", format("{oozie_user}:{user_group}"),
oozie_ext_zip_target_path),
sudo=True)
File(oozie_ext_zip_target_path, mode=0644)
break
if not found_at_least_one_oozie_ext_file:
raise Fail(
"Unable to find any Oozie source extension files from the following paths {0}"
.format(source_ext_zip_paths))
# Redownload jdbc driver to a new current location
oozie.download_database_library_if_needed()
# get the upgrade version in the event that it's needed
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None or len(
upgrade_stack) < 2 or upgrade_stack[1] is None:
raise Fail(
"Unable to determine the stack that is being upgraded to or downgraded to."
)
stack_version = upgrade_stack[1]
# copy the Falcon JAR if needed; falcon has not upgraded yet, so we must
# use the versioned falcon directory
if params.has_falcon_host:
versioned_falcon_jar_directory = "{0}/{1}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(
params.stack_root, stack_version)
Logger.info("Copying {0} to {1}".format(
versioned_falcon_jar_directory, params.oozie_libext_dir))
Execute(
format(
'{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}'
))
Execute(
format(
'{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'
))
