def actionexecute(self, env):
config = Script.get_config()
structured_output = {}
try:
repo_info_json = config['hostLevelParams']['repo_info']
repo_info_dict = json.loads(repo_info_json)
for item in repo_info_dict["repositories"]:
base_url = item["base_url"]
repo_name = item["repo_name"]
repo_id = item["repo_id"]
repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else repo_ubuntu
ubuntu_components = [repo_name] + self.UBUNTU_REPO_COMPONENTS_POSTFIX
Repository(repo_id,
action = "create",
base_url = base_url,
mirror_list = None,
repo_file_name = repo_name,
repo_template = template,
components = ubuntu_components, # ubuntu specific
)
structured_output["repo_update"] = {"exit_code" : 0, "message": format("Repository files successfully updated!")}
except Exception, exception:
Logger.logger.exception("ERROR: There was an unexpected error while updating repositories")
raise Fail("Failed to update repo files!")
def set_uid(user, user_dirs):
"""
user_dirs - comma separated directories
"""
import params
File(format("{tmp_dir}/changeUid.sh"),
content=StaticFile("changeToSecureUid.sh"),
mode=0555)
ignore_groupsusers_create_str = str(
params.ignore_groupsusers_create).lower()
Execute(
format("{tmp_dir}/changeUid.sh {user} {user_dirs}"),
not_if=format(
"(test $(id -u {user}) -gt 1000) || ({ignore_groupsusers_create_str})"
))
def actionexecute(self, env):
config = Script.get_config()
structured_output = {}
try:
repo_info = config['repositoryFile']
for item in repo_info["repositories"]:
base_url = item["baseUrl"]
repo_name = item["repoName"]
repo_id = item["repoId"]
distribution = item["distribution"] if "distribution" in item else None
components = item["components"] if "components" in item else None
repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else repo_ubuntu
ubuntu_components = [distribution if distribution else repo_name] + \
[components.replace(",", " ") if components else self.UBUNTU_REPO_COMPONENTS_POSTFIX]
Repository(repo_id,
action = "create",
base_url = base_url,
mirror_list = None,
repo_file_name = repo_name,
repo_template = template,
components = ubuntu_components, # ubuntu specific
)
structured_output["repo_update"] = {"exit_code" : 0, "message": format("Repository files successfully updated!")}
except Exception, exception:
Logger.logger.exception("ERROR: There was an unexpected error while updating repositories")
raise Fail("Failed to update repo files!")
def setup_livy(env, type, upgrade_type=None, action=None):
import params
Directory([params.livy2_pid_dir, params.livy2_log_dir],
owner=params.livy2_user,
group=params.user_group,
mode=0775,
create_parents=True)
if type == 'server' and action == 'config':
params.HdfsResource(params.livy2_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.livy2_user,
mode=0775)
params.HdfsResource(None, action="execute")
# create livy-env.sh in etc/conf dir
File(
os.path.join(params.livy2_conf, 'livy-env.sh'),
owner=params.livy2_user,
group=params.livy2_group,
content=InlineTemplate(params.livy2_env_sh),
mode=0644,
)
# create livy.conf in etc/conf dir
PropertiesFile(
format("{livy2_conf}/livy.conf"),
properties=params.config['configurations']['livy2-conf'],
key_value_delimiter=" ",
owner=params.livy2_user,
group=params.livy2_group,
)
# create log4j.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'log4j.properties'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_log4j_properties,
mode=0644,
)
# create spark-blacklist.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'spark-blacklist.conf'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_spark_blacklist_properties,
mode=0644,
)
Directory(
params.livy2_logs_dir,
owner=params.livy2_user,
group=params.livy2_group,
mode=0755,
)
def actionexecute(self, env):
config = Script.get_config()
structured_output = {}
cmd = self.get_clearcache_cmd()
Logger.info("Clearing repository cache")
code, output = shell.call(cmd, sudo = True)
if 0 == code:
structured_output["clear_repocache"] = {"exit_code" : 0, "message": format("Repository cache successfully cleared!")}
else:
structured_output["clear_repocache"] = {"exit_code": code, "message": "Failed to clear repository cache! {0}".format(str(output))}
self.put_structured_out(structured_output)
def remove_stack_version(self, structured_output, version):
# check simlinks not refer to version for remove
self.check_no_symlink_to_version(structured_output, version)
packages_to_remove = self.get_packages_to_remove(version)
for package in packages_to_remove:
Package(package, action="remove")
self.remove_stack_folder(structured_output, version)
structured_output["remove_previous_stacks"] = {
"exit_code":
0,
"message":
format("Stack version {0} successfully removed!".format(version))
}
self.put_structured_out(structured_output)
def install_jce_policy():
import params
jcePolicyInfo = JcePolicyInfo(params.java_home)
if jcePolicyInfo.is_unlimited_key_jce_policy():
major, minor = java_version()
Logger.info(
"The unlimited key JCE policy is required, and appears to have been installed."
)
java_security_dir = format("{java_home}/jre/lib/security")
Logger.debug("Removing existing JCE policy JAR files: {0}.".format(
java_security_dir))
File(format("{java_security_dir}/US_export_policy.jar"),
action="delete")
File(format("{java_security_dir}/local_policy.jar"), action="delete")
src_url = jce_zip_target = ''
if minor == 7: # jdk7
src_url = 'http://yum.example.com/hadoop/UnlimitedJCEPolicyJDK7.zip'
jce_zip_target = '/tmp/UnlimitedJCEPolicyJDK7.zip'
elif minor == 8: # jdk8
src_url = 'http://yum.example.com/hadoop/jce_policy-8.zip'
jce_zip_target = '/tmp/jce_policy-8.zip'
if src_url and jce_zip_target:
Execute('wget ' + src_url + ' -O ' + jce_zip_target)
Logger.debug(
"Unzipping the unlimited key JCE policy files from {0} into {1}."
.format(jce_zip_target, java_security_dir))
extract_cmd = ("unzip", "-o", "-j", "-q", jce_zip_target, "-d",
java_security_dir)
Execute(
extract_cmd,
only_if=format(
"test -e {java_security_dir} && test -f {jce_zip_target}"),
path=['/bin/', '/usr/bin'],
sudo=True)
def actionexecute(self, env):
config = Script.get_config()
structured_output = {}
try:
repo_info_json = config['hostLevelParams']['repo_info']
repo_info_dict = json.loads(repo_info_json)
for item in repo_info_dict["repositories"]:
base_url = item["base_url"]
repo_name = item["repo_name"]
repo_id = item["repo_id"]
repo_rhel_suse = config['configurations']['cluster-env'][
'repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env'][
'repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_suse_family(
) or OSCheck.is_redhat_family() else repo_ubuntu
ubuntu_components = [repo_name
] + self.UBUNTU_REPO_COMPONENTS_POSTFIX
Repository(
repo_id,
action="create",
base_url=base_url,
mirror_list=None,
repo_file_name=repo_name,
repo_template=template,
components=ubuntu_components, # ubuntu specific
)
structured_output["repo_update"] = {
"exit_code": 0,
"message": format("Repository files successfully updated!")
}
except Exception, exception:
Logger.logger.exception(
"ERROR: There was an unexpected error while updating repositories"
)
raise Fail("Failed to update repo files!")
def modify_rack_awareness():
import params
hostname = params.config["hostname"].lower()
ip = socket.gethostbyname(socket.gethostname())
try:
response = urllib2.urlopen(CONST_RACK_INFO_URL + ip)
rack_info = response.read()
rack_info = json.loads(rack_info)
if isinstance(rack_info, list):
topology_item = '/default/default/default'
else:
datacenter = rack_info.get(ip).get('datacenter') or 'default'
switch = rack_info.get(ip).get('switch') or 'default'
rack = rack_info.get(ip).get('rack') or 'default'
topology_item = '/' + datacenter.replace(
'/', '').lower() + '/' + switch.replace(
'/', '').lower() + '/' + rack.replace('/', '').lower()
ambari_server_host = params.config['clusterHostInfo'][
'ambari_server_host'][0]
ambari_server_port = params.config['clusterHostInfo'][
'ambari_server_port'][0]
ambari_server_use_ssl = params.config['clusterHostInfo'][
'ambari_server_use_ssl'][0] == 'true'
ambari_server_protocol = 'https' if ambari_server_use_ssl else 'http'
cluster_name = params.config['clusterName']
ambari_server_auth_host_url = format(
'{ambari_server_protocol}://{ambari_server_host}:{ambari_server_port}'
)
rack_update_url = ambari_server_auth_host_url + '/api/v1/clusters/' + cluster_name + '/hosts'
rack_update = '{"RequestInfo":{"context":"Set Rack","query":"Hosts/host_name.in(' + hostname + ')"},"Body":{"Hosts":{"rack_info":"' + topology_item + '"}}}'
Execute(
"curl -u 'ambari-qa:example.com' -H 'X-Requested-By:ambari' -i " +
rack_update_url + " -X PUT -d '" + rack_update + "'")
except urllib2.HTTPError as e:
print "can not get %s rack_info from cmdb" % ip
def execute_db_connection_check(self, config, tmp_dir):
Logger.info("DB connection check started.")
# initialize needed data
ambari_server_hostname = config['commandParams']['ambari_server_host']
check_db_connection_jar_name = "DBConnectionVerification.jar"
jdk_location = config['commandParams']['jdk_location']
java_home = config['commandParams']['java_home']
db_name = config['commandParams']['db_name']
no_jdbc_error_message = None
if db_name == DB_MYSQL:
jdbc_driver_mysql_name = default(
"/hostLevelParams/custom_mysql_jdbc_name", None)
if not jdbc_driver_mysql_name:
no_jdbc_error_message = "The MySQL JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=mysql --jdbc-driver=/path/to/jdbc_driver'."
else:
jdbc_url = jdk_location + jdbc_driver_mysql_name
jdbc_driver_class = JDBC_DRIVER_CLASS_MYSQL
jdbc_name = jdbc_driver_mysql_name
elif db_name == DB_ORACLE:
jdbc_driver_oracle_name = default(
"/hostLevelParams/custom_oracle_jdbc_name", None)
if not jdbc_driver_oracle_name:
no_jdbc_error_message = "The Oracle JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=oracle --jdbc-driver=/path/to/jdbc_driver'."
else:
jdbc_url = jdk_location + jdbc_driver_oracle_name
jdbc_driver_class = JDBC_DRIVER_CLASS_ORACLE
jdbc_name = jdbc_driver_oracle_name
elif db_name == DB_POSTGRESQL:
jdbc_driver_postgres_name = default(
"/hostLevelParams/custom_postgres_jdbc_name", None)
if not jdbc_driver_postgres_name:
no_jdbc_error_message = "The Postgres JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=postgres --jdbc-driver=/path/to/jdbc_driver'."
else:
jdbc_url = jdk_location + jdbc_driver_postgres_name
jdbc_driver_class = JDBC_DRIVER_CLASS_POSTGRESQL
jdbc_name = jdbc_driver_postgres_name
elif db_name == DB_MSSQL:
jdbc_driver_mssql_name = default(
"/hostLevelParams/custom_mssql_jdbc_name", None)
if not jdbc_driver_mssql_name:
no_jdbc_error_message = "The MSSQL JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=mssql --jdbc-driver=/path/to/jdbc_driver'."
else:
jdbc_url = jdk_location + jdbc_driver_mssql_name
jdbc_driver_class = JDBC_DRIVER_CLASS_MSSQL
jdbc_name = jdbc_driver_mssql_name
elif db_name == DB_SQLA:
jdbc_driver_sqla_name = default(
"/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
if not jdbc_driver_sqla_name:
no_jdbc_error_message = "The SQLAnywhere JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=sqlanywhere --jdbc-driver=/path/to/jdbc_driver'."
else:
jdbc_url = jdk_location + jdbc_driver_sqla_name
jdbc_driver_class = JDBC_DRIVER_CLASS_SQLA
jdbc_name = jdbc_driver_sqla_name
else:
no_jdbc_error_message = format(
"'{db_name}' database type not supported.")
if no_jdbc_error_message:
Logger.warning(no_jdbc_error_message)
db_connection_check_structured_output = {
"exit_code": 1,
"message": no_jdbc_error_message
}
return db_connection_check_structured_output
db_connection_url = config['commandParams']['db_connection_url']
user_name = config['commandParams']['user_name']
user_passwd = config['commandParams']['user_passwd']
agent_cache_dir = os.path.abspath(
config["hostLevelParams"]["agentCacheDir"])
check_db_connection_url = jdk_location + check_db_connection_jar_name
jdbc_path = os.path.join(agent_cache_dir, jdbc_name)
class_path_delimiter = ":"
if db_name == DB_SQLA:
jdbc_jar_path = agent_cache_dir + JDBC_DRIVER_SQLA_JAR_PATH_IN_ARCHIVE
java_library_path = agent_cache_dir + JARS_PATH_IN_ARCHIVE_SQLA + class_path_delimiter + agent_cache_dir + \
LIBS_PATH_IN_ARCHIVE_SQLA
else:
jdbc_jar_path = jdbc_path
java_library_path = agent_cache_dir
check_db_connection_path = os.path.join(agent_cache_dir,
check_db_connection_jar_name)
java_bin = "java"
if OSCheck.is_windows_family():
java_bin = "java.exe"
class_path_delimiter = ";"
java_exec = os.path.join(java_home, "bin", java_bin)
if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \
or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec):
message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \
"\n"
Logger.warning(message)
db_connection_check_structured_output = {
"exit_code": 1,
"message": message
}
return db_connection_check_structured_output
environment = {"no_proxy": format("{ambari_server_hostname}")}
# download and install java if it doesn't exists
if not os.path.isfile(java_exec):
jdk_name = config['commandParams']['jdk_name']
jdk_url = "{0}/{1}".format(jdk_location, jdk_name)
jdk_download_target = os.path.join(agent_cache_dir, jdk_name)
java_dir = os.path.dirname(java_home)
try:
download_file(jdk_url, jdk_download_target)
except Exception, e:
message = "Error downloading JDK from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
Logger.exception(message)
db_connection_check_structured_output = {
"exit_code": 1,
"message": message
}
return db_connection_check_structured_output
if jdk_name.endswith(".exe"):
install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format(
os_utils.quote_path(jdk_download_target),
os_utils.quote_path(java_home),
)
install_path = [java_dir]
try:
Execute(install_cmd, path=install_path)
except Exception, e:
message = "Error installing java.\n" + str(e)
Logger.exception(message)
db_connection_check_structured_output = {
"exit_code": 1,
"message": message
}
return db_connection_check_structured_output
Execute(install_cmd, path=install_path)
except Exception, e:
message = "Error installing java.\n" + str(e)
Logger.exception(message)
db_connection_check_structured_output = {
"exit_code": 1,
"message": message
}
return db_connection_check_structured_output
else:
tmp_java_dir = tempfile.mkdtemp(prefix="jdk_tmp_", dir=tmp_dir)
sudo = AMBARI_SUDO_BINARY
if jdk_name.endswith(".bin"):
chmod_cmd = ("chmod", "+x", jdk_download_target)
install_cmd = format(
"cd {tmp_java_dir} && echo A | {jdk_download_target} -noregister && {sudo} cp -rp {tmp_java_dir}/* {java_dir}"
)
elif jdk_name.endswith(".gz"):
chmod_cmd = ("chmod", "a+x", java_dir)
install_cmd = format(
"cd {tmp_java_dir} && tar -xf {jdk_download_target} && {sudo} cp -rp {tmp_java_dir}/* {java_dir}"
)
try:
Directory(java_dir)
Execute(chmod_cmd,
not_if=format("test -e {java_exec}"),
sudo=True)
Execute(install_cmd, not_if=format("test -e {java_exec}"))
File(format("{java_home}/bin/java"),
mode=0755,
cd_access="a")
def execute_db_connection_check(self, config, tmp_dir):
print "DB connection check started."
# initialize needed data
ambari_server_hostname = config['commandParams']['ambari_server_host']
check_db_connection_jar_name = "DBConnectionVerification.jar"
jdk_location = config['commandParams']['jdk_location']
java_home = config['commandParams']['java_home']
db_name = config['commandParams']['db_name']
if db_name == DB_MYSQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MYSQL
jdbc_driver_class = JDBC_DRIVER_CLASS_MYSQL
jdbc_name = JDBC_DRIVER_SYMLINK_MYSQL
elif db_name == DB_ORACLE:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_ORACLE
jdbc_driver_class = JDBC_DRIVER_CLASS_ORACLE
jdbc_name = JDBC_DRIVER_SYMLINK_ORACLE
elif db_name == DB_POSTGRESQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_POSTGRESQL
jdbc_driver_class = JDBC_DRIVER_CLASS_POSTGRESQL
jdbc_name = JDBC_DRIVER_SYMLINK_POSTGRESQL
elif db_name == DB_MSSQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MSSQL
jdbc_driver_class = JDBC_DRIVER_CLASS_MSSQL
jdbc_name = JDBC_DRIVER_SYMLINK_MSSQL
elif db_name == DB_SQLA:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_SQLA
jdbc_driver_class = JDBC_DRIVER_CLASS_SQLA
jdbc_name = JDBC_DRIVER_SYMLINK_SQLA
db_connection_url = config['commandParams']['db_connection_url']
user_name = config['commandParams']['user_name']
user_passwd = config['commandParams']['user_passwd']
agent_cache_dir = os.path.abspath(config["hostLevelParams"]["agentCacheDir"])
check_db_connection_url = jdk_location + check_db_connection_jar_name
jdbc_path = os.path.join(agent_cache_dir, jdbc_name)
class_path_delimiter = ":"
if db_name == DB_SQLA:
jdbc_jar_path = agent_cache_dir + JDBC_DRIVER_SQLA_JAR_PATH_IN_ARCHIVE
java_library_path = agent_cache_dir + JARS_PATH_IN_ARCHIVE_SQLA + class_path_delimiter + agent_cache_dir + \
LIBS_PATH_IN_ARCHIVE_SQLA
else:
jdbc_jar_path = jdbc_path
java_library_path = agent_cache_dir
check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name)
java_bin = "java"
if OSCheck.is_windows_family():
java_bin = "java.exe"
class_path_delimiter = ";"
java_exec = os.path.join(java_home, "bin",java_bin)
if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \
or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec):
message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \
"\n"
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
environment = { "no_proxy": format("{ambari_server_hostname}") }
# download and install java if it doesn't exists
if not os.path.isfile(java_exec):
jdk_name = config['commandParams']['jdk_name']
jdk_url = "{0}/{1}".format(jdk_location, jdk_name)
jdk_download_target = os.path.join(agent_cache_dir, jdk_name)
java_dir = os.path.dirname(java_home)
try:
download_file(jdk_url, jdk_download_target)
except Exception, e:
message = "Error downloading JDK from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
if jdk_name.endswith(".exe"):
install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format(
os_utils.quote_path(jdk_download_target), os_utils.quote_path(java_home),
)
install_path = [java_dir]
try:
Execute(install_cmd, path = install_path)
except Exception, e:
message = "Error installing java.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
return db_connection_check_structured_output
if jdk_name.endswith(".exe"):
install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format(
os_utils.quote_path(jdk_download_target), os_utils.quote_path(java_home),
)
install_path = [java_dir]
try:
Execute(install_cmd, path = install_path)
except Exception, e:
message = "Error installing java.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
else:
tmp_java_dir = format("{tmp_dir}/jdk")
sudo = AMBARI_SUDO_BINARY
if jdk_name.endswith(".bin"):
chmod_cmd = ("chmod", "+x", jdk_download_target)
install_cmd = format("mkdir -p {tmp_java_dir} && cd {tmp_java_dir} && echo A | {jdk_download_target} -noregister && {sudo} cp -rp {tmp_java_dir}/* {java_dir}")
elif jdk_name.endswith(".gz"):
chmod_cmd = ("chmod","a+x", java_dir)
install_cmd = format("mkdir -p {tmp_java_dir} && cd {tmp_java_dir} && tar -xf {jdk_download_target} && {sudo} cp -rp {tmp_java_dir}/* {java_dir}")
try:
Directory(java_dir)
Execute(chmod_cmd, not_if = format("test -e {java_exec}"), sudo = True)
Execute(install_cmd, not_if = format("test -e {java_exec}"))
File(format("{java_home}/bin/java"), mode=0755, cd_access="a")
Execute(("chown","-R", getpass.getuser(), java_home), sudo = True)
except Exception, e:
message = "Error installing java.\n" + str(e)
def setup_java():
"""
Installs jdk using specific params, that comes from ambari-server
"""
import params
java_exec = format("{java_home}/bin/java")
if not os.path.isfile(java_exec):
jdk_curl_target = format("{tmp_dir}/{jdk_name}")
java_dir = os.path.dirname(params.java_home)
Directory(
params.artifact_dir,
create_parents=True,
)
File(jdk_curl_target,
content=DownloadSource(format("{jdk_location}/{jdk_name}")),
not_if=format("test -f {jdk_curl_target}"))
File(
jdk_curl_target,
mode=0755,
)
tmp_java_dir = tempfile.mkdtemp(prefix="jdk_tmp_", dir=params.tmp_dir)
try:
if params.jdk_name.endswith(".bin"):
chmod_cmd = ("chmod", "+x", jdk_curl_target)
install_cmd = format(
"cd {tmp_java_dir} && echo A | {jdk_curl_target} -noregister && {sudo} cp -rp {tmp_java_dir}/* {java_dir}"
)
elif params.jdk_name.endswith(".gz"):
chmod_cmd = ("chmod", "a+x", java_dir)
install_cmd = format(
"cd {tmp_java_dir} && tar -xf {jdk_curl_target} && {sudo} /bin/cp -rp {tmp_java_dir}/* {java_dir}"
)
Directory(java_dir)
Execute(
chmod_cmd,
sudo=True,
)
Execute(install_cmd, )
java_version_dir = os.listdir(tmp_java_dir)[0]
Execute("ln -s %s %s" %
(java_dir + '/' + java_version_dir, params.java_home))
finally:
Directory(tmp_java_dir, action="delete")
File(
format("{java_home}/bin/java"),
mode=0755,
cd_access="a",
)
Execute(
('chmod', '-R', '755', params.java_home),
sudo=True,
)
message = "Error downloading DBConnectionVerification.jar from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
# download jdbc driver from ambari-server resources
try:
download_file(jdbc_url, jdbc_path)
if db_name == DB_MSSQL and OSCheck.is_windows_family():
jdbc_auth_path = os.path.join(agent_cache_dir, JDBC_AUTH_SYMLINK_MSSQL)
jdbc_auth_url = jdk_location + JDBC_AUTH_SYMLINK_MSSQL
download_file(jdbc_auth_url, jdbc_auth_path)
except Exception, e:
message = format("Error: Ambari Server cannot download the database JDBC driver and is unable to test the " \
"database connection. You must run tbds-server setup --jdbc-db={db_name} " \
"--jdbc-driver=/path/to/your/{db_name}/driver.jar on the Ambari Server host to make the JDBC " \
"driver available for download and to enable testing the database connection.\n") + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
# try to connect to db
db_connection_check_command = format("{java_exec} -cp {check_db_connection_path}{class_path_delimiter}" \
"{jdbc_path} -Djava.library.path={agent_cache_dir} org.apache.ambari.server.DBConnectionVerification \"{db_connection_url}\" " \
"{user_name} {user_passwd!p} {jdbc_driver}")
print "INFO db_connection_check_command: " + db_connection_check_command
process = subprocess.Popen(db_connection_check_command,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
def setup_livy(env, type, upgrade_type=None, action=None):
import params
Directory([params.livy2_pid_dir, params.livy2_log_dir],
owner=params.livy2_user,
group=params.user_group,
mode=0775,
cd_access='a',
create_parents=True)
if type == 'server' and action == 'config':
params.HdfsResource(params.livy2_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.livy2_user,
mode=0775)
params.HdfsResource(None, action="execute")
if params.livy2_recovery_store == 'filesystem':
params.HdfsResource(params.livy2_recovery_dir,
type="directory",
action="create_on_execute",
owner=params.livy2_user,
mode=0700)
params.HdfsResource(None, action="execute")
generate_logfeeder_input_config(
'spark2',
Template("input.config-spark2.json.j2", extra_imports=[default]))
# create livy-env.sh in etc/conf dir
File(
os.path.join(params.livy2_conf, 'livy-env.sh'),
owner=params.livy2_user,
group=params.livy2_group,
content=InlineTemplate(params.livy2_env_sh),
mode=0644,
)
# create livy-client.conf in etc/conf dir
PropertiesFile(
format("{livy2_conf}/livy-client.conf"),
properties=params.config['configurations']['livy2-client-conf'],
key_value_delimiter=" ",
owner=params.livy2_user,
group=params.livy2_group,
)
# create livy.conf in etc/conf dir
PropertiesFile(
format("{livy2_conf}/livy.conf"),
properties=params.config['configurations']['livy2-conf'],
key_value_delimiter=" ",
owner=params.livy2_user,
group=params.livy2_group,
)
# create log4j.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'log4j.properties'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_log4j_properties,
mode=0644,
)
# create spark-blacklist.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'spark-blacklist.conf'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_spark_blacklist_properties,
mode=0644,
)
Directory(
params.livy2_logs_dir,
owner=params.livy2_user,
group=params.livy2_group,
mode=0755,
)
)
try:
Execute(install_cmd, path=["/bin", "/usr/bin/"])
except Exception, e:
message = "Error installing java.\n" + str(e)
print message
db_connection_check_structured_output = {
"exit_code": "1",
"message": message
}
return db_connection_check_structured_output
try:
cmd = format(
"/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf "
"--retry 5 {jdk_location}{check_db_connection_jar_name} "
"-o {check_db_connection_jar_name}'")
Execute(
cmd,
not_if=format(
"[ -f /usr/lib/ambari-agent/{check_db_connection_jar_name}]"
),
environment=environment)
except Exception, e:
message = "Error downloading DBConnectionVerification.jar from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
print message
db_connection_check_structured_output = {
"exit_code": "1",
"message": message
}
message = "Error downloading DBConnectionVerification.jar from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
# download jdbc driver from ambari-server resources
try:
download_file(jdbc_url, jdbc_path)
if db_name == DB_MSSQL and OSCheck.is_windows_family():
jdbc_auth_path = os.path.join(agent_cache_dir, JDBC_AUTH_SYMLINK_MSSQL)
jdbc_auth_url = jdk_location + JDBC_AUTH_SYMLINK_MSSQL
download_file(jdbc_auth_url, jdbc_auth_path)
except Exception, e:
message = format("Error: Ambari Server cannot download the database JDBC driver and is unable to test the " \
"database connection. You must run ambari-server setup --jdbc-db={db_name} " \
"--jdbc-driver=/path/to/your/{db_name}/driver.jar on the Ambari Server host to make the JDBC " \
"driver available for download and to enable testing the database connection.\n") + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
# try to connect to db
db_connection_check_command = format("{java_exec} -cp {check_db_connection_path}{class_path_delimiter}" \
"{jdbc_path} -Djava.library.path={agent_cache_dir} org.apache.ambari.server.DBConnectionVerification \"{db_connection_url}\" " \
"{user_name} {user_passwd!p} {jdbc_driver}")
print "INFO db_connection_check_command: " + db_connection_check_command
process = subprocess.Popen(db_connection_check_command,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
try:
Execute(install_cmd, path=["/bin", "/usr/bin/"])
except Exception, e:
message = "Error installing java.\n" + str(e)
print message
db_connection_check_structured_output = {
"exit_code": 1,
"message": message
}
return db_connection_check_structured_output
# download DBConnectionVerification.jar from ambari-server resources
try:
cmd = format(
"/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf "
"--retry 5 {jdk_location}{check_db_connection_jar_name} "
"-o {check_db_connection_jar_name}'")
Execute(
cmd,
not_if=format(
"[ -f /usr/lib/ambari-agent/{check_db_connection_jar_name}]"
),
environment=environment)
except Exception, e:
message = "Error downloading DBConnectionVerification.jar from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
print message
db_connection_check_structured_output = {
"exit_code": 1,
"message": message
}
def execute_db_connection_check(self, config, tmp_dir):
print "DB connection check started."
# initialize needed data
ambari_server_hostname = config['commandParams']['ambari_server_host']
check_db_connection_jar_name = "DBConnectionVerification.jar"
jdk_location = config['commandParams']['jdk_location']
java64_home = config['commandParams']['java_home']
db_name = config['commandParams']['db_name']
if db_name == DB_MYSQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MYSQL
jdbc_driver = JDBC_DRIVER_MYSQL
jdbc_name = JDBC_DRIVER_SYMLINK_MYSQL
elif db_name == DB_ORACLE:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_ORACLE
jdbc_driver = JDBC_DRIVER_ORACLE
jdbc_name = JDBC_DRIVER_SYMLINK_ORACLE
elif db_name == DB_POSTGRESQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_POSTGRESQL
jdbc_driver = JDBC_DRIVER_POSTGRESQL
jdbc_name = JDBC_DRIVER_SYMLINK_POSTGRESQL
elif db_name == DB_MSSQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MSSQL
jdbc_driver = JDBC_DRIVER_MSSQL
jdbc_name = JDBC_DRIVER_SYMLINK_MSSQL
db_connection_url = config['commandParams']['db_connection_url']
user_name = config['commandParams']['user_name']
user_passwd = config['commandParams']['user_passwd']
agent_cache_dir = os.path.abspath(config["hostLevelParams"]["agentCacheDir"])
check_db_connection_url = jdk_location + check_db_connection_jar_name
jdbc_path = os.path.join(agent_cache_dir, jdbc_name)
check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name)
java_bin = "java"
class_path_delimiter = ":"
if OSCheck.is_windows_family():
java_bin = "java.exe"
class_path_delimiter = ";"
java_exec = os.path.join(java64_home, "bin",java_bin)
if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \
or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec):
message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \
"\n"
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
environment = { "no_proxy": format("{ambari_server_hostname}") }
# download and install java if it doesn't exists
if not os.path.isfile(java_exec):
jdk_name = config['commandParams']['jdk_name']
jdk_url = "{0}/{1}".format(jdk_location, jdk_name)
jdk_download_target = os.path.join(agent_cache_dir, jdk_name)
java_dir = os.path.dirname(java64_home)
try:
download_file(jdk_url, jdk_download_target)
except Exception, e:
message = "Error downloading JDK from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
if jdk_name.endswith(".bin"):
install_cmd = format("mkdir -p {java_dir} ; chmod +x {jdk_download_target}; cd {java_dir} ; echo A | " \
"{jdk_curl_target} -noregister > /dev/null 2>&1")
install_path = ["/bin","/usr/bin/"]
elif jdk_name.endswith(".gz"):
install_cmd = format("mkdir -p {java_dir} ; cd {java_dir} ; tar -xf {jdk_download_target} > /dev/null 2>&1")
install_path = ["/bin","/usr/bin/"]
elif jdk_name.endswith(".exe"):
install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format(
os_utils.quote_path(jdk_download_target), os_utils.quote_path(java64_home),
)
install_path = [java_dir]
try:
Execute(install_cmd, path = install_path)
except Exception, e:
message = "Error installing java.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
"{jdk_curl_target} -noregister > /dev/null 2>&1")
elif jdk_name.endswith(".gz"):
install_cmd = format("mkdir -p {java_dir} ; cd {java_dir} ; tar -xf {jdk_curl_target} > /dev/null 2>&1")
try:
Execute(install_cmd, path = ["/bin","/usr/bin/"])
except Exception, e:
message = "Error installing java.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
# download DBConnectionVerification.jar from ambari-server resources
try:
cmd = format("/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf "
"--retry 5 {jdk_location}{check_db_connection_jar_name} "
"-o {check_db_connection_jar_name}'")
Execute(cmd, not_if=format("[ -f /usr/lib/ambari-agent/{check_db_connection_jar_name}]"), environment = environment)
except Exception, e:
message = "Error downloading DBConnectionVerification.jar from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
# download jdbc driver from ambari-server resources
try:
cmd = format("/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf "
"--retry 5 {jdbc_url} -o {jdbc_name}'")
Execute(cmd, not_if=format("[ -f /usr/lib/ambari-agent/{jdbc_name}]"), environment = environment)
def execute_db_connection_check(self, config):
print "DB connection check started."
# initialize needed data
ambari_server_hostname = config['commandParams']['ambari_server_host']
check_db_connection_jar_name = "DBConnectionVerification.jar"
jdk_location = config['commandParams']['jdk_location']
java64_home = config['commandParams']['java_home']
db_name = config['commandParams']['db_name']
if db_name == DB_MYSQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MYSQL
jdbc_driver = JDBC_DRIVER_MYSQL
jdbc_name = JDBC_DRIVER_SYMLINK_MYSQL
elif db_name == DB_ORACLE:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_ORACLE
jdbc_driver = JDBC_DRIVER_ORACLE
jdbc_name = JDBC_DRIVER_SYMLINK_ORACLE
elif db_name == DB_POSTGRESQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_POSTGRESQL
jdbc_driver = JDBC_DRIVER_POSTGRESQL
jdbc_name = JDBC_DRIVER_SYMLINK_POSTGRESQL
db_connection_url = config['commandParams']['db_connection_url']
user_name = config['commandParams']['user_name']
user_passwd = config['commandParams']['user_passwd']
java_exec = os.path.join(java64_home, "bin", "java")
if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \
or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec):
message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \
"\n"
print message
db_connection_check_structured_output = {
"exit_code": "1",
"message": message
}
return db_connection_check_structured_output
environment = {"no_proxy": format("{ambari_server_hostname}")}
artifact_dir = "/tmp/HDP-artifacts/"
jdk_name = config['commandParams']['jdk_name']
jdk_curl_target = format("{artifact_dir}/{jdk_name}")
java_dir = os.path.dirname(java64_home)
# download DBConnectionVerification.jar from ambari-server resources
if not os.path.isfile(java_exec):
try:
Execute(format(
"mkdir -p {artifact_dir} ; curl -kf "
"--retry 10 {jdk_location}/{jdk_name} -o {jdk_curl_target}"
),
path=["/bin", "/usr/bin/"],
environment=environment)
except Exception, e:
message = "Error downloading JDK from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
print message
db_connection_check_structured_output = {
"exit_code": "1",
"message": message
}
return db_connection_check_structured_output
if jdk_name.endswith(".bin"):
install_cmd = format("mkdir -p {java_dir} ; chmod +x {jdk_curl_target}; cd {java_dir} ; echo A | " \
"{jdk_curl_target} -noregister > /dev/null 2>&1")
elif jdk_name.endswith(".gz"):
install_cmd = format(
"mkdir -p {java_dir} ; cd {java_dir} ; tar -xf {jdk_curl_target} > /dev/null 2>&1"
)
try:
Execute(install_cmd, path=["/bin", "/usr/bin/"])
except Exception, e:
message = "Error installing java.\n" + str(e)
print message
db_connection_check_structured_output = {
"exit_code": "1",
"message": message
}
return db_connection_check_structured_output
def execute_db_connection_check(self, config, tmp_dir):
print "DB connection check started."
# initialize needed data
ambari_server_hostname = config['commandParams']['ambari_server_host']
check_db_connection_jar_name = "DBConnectionVerification.jar"
jdk_location = config['commandParams']['jdk_location']
java64_home = config['commandParams']['java_home']
db_name = config['commandParams']['db_name']
if db_name == DB_MYSQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MYSQL
jdbc_driver = JDBC_DRIVER_MYSQL
jdbc_name = JDBC_DRIVER_SYMLINK_MYSQL
elif db_name == DB_ORACLE:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_ORACLE
jdbc_driver = JDBC_DRIVER_ORACLE
jdbc_name = JDBC_DRIVER_SYMLINK_ORACLE
elif db_name == DB_POSTGRESQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_POSTGRESQL
jdbc_driver = JDBC_DRIVER_POSTGRESQL
jdbc_name = JDBC_DRIVER_SYMLINK_POSTGRESQL
db_connection_url = config['commandParams']['db_connection_url']
user_name = config['commandParams']['user_name']
user_passwd = config['commandParams']['user_passwd']
java_exec = os.path.join(java64_home, "bin","java")
if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \
or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec):
message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \
"\n"
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
environment = { "no_proxy": format("{ambari_server_hostname}") }
artifact_dir = format("{tmp_dir}/AMBARI-artifacts/")
java_dir = os.path.dirname(java64_home)
# download and install java if it doesn't exists
if not os.path.isfile(java_exec):
try:
jdk_name = config['commandParams']['jdk_name']
jdk_curl_target = format("{artifact_dir}/{jdk_name}")
Execute(format("mkdir -p {artifact_dir} ; curl -kf "
"--retry 10 {jdk_location}/{jdk_name} -o {jdk_curl_target}"),
path = ["/bin","/usr/bin/"],
environment = environment)
except Exception, e:
message = "Error downloading JDK from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
if jdk_name.endswith(".bin"):
install_cmd = format("mkdir -p {java_dir} ; chmod +x {jdk_curl_target}; cd {java_dir} ; echo A | " \
"{jdk_curl_target} -noregister > /dev/null 2>&1")
elif jdk_name.endswith(".gz"):
install_cmd = format("mkdir -p {java_dir} ; cd {java_dir} ; tar -xf {jdk_curl_target} > /dev/null 2>&1")
try:
Execute(install_cmd, path = ["/bin","/usr/bin/"])
except Exception, e:
message = "Error installing java.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
