def action_install(self):
name = self.resource.msi_name
msi_file_path = name
dict_args = self.resource.dict_args
list_args = self.resource.list_args
working_dir = os.path.abspath(Script.get_config()["hostLevelParams"]["agentCacheDir"])
http_source = self.resource.http_source
# name can be a path to file in local file system
msi_filename = os.path.split(name)[1]
log_file_path = os.path.join(working_dir, msi_filename) + ".log"
marker_file = os.path.join(working_dir, msi_filename) + ".installed"
# build string from passed arguments to Msi resource
dict_args_str = ' ALLUSERS="1"'
for k, v in dict_args.iteritems():
dict_args_str += " " + str(k)+"="+str(v)
list_args_str = ''
for a in list_args:
list_args_str += " /" + str(a)
# if http source present we download msi and then execute it
if http_source:
download_url = urlparse.urljoin(http_source, name)
msi_file_path = os.path.join(working_dir, msi_filename)
download_file(download_url, msi_file_path)
if not os.path.exists(marker_file):
Execute(MsiProvider.MSI_INSTALL_COMMAND.format(msi_file_path=msi_file_path,
log_file_path=log_file_path,
dict_args_str=dict_args_str,
list_args_str=list_args_str).rstrip())
# writing marker file to not install new msi later
open(marker_file,"w").close()
def install_windows_msi(msi_url, save_dir, save_file, hadoop_user,
hadoop_password, stack_version):
global _working_dir
_working_dir = save_dir
save_dir = os.path.abspath(save_dir)
msi_save_dir = save_dir
# system wide lock to prevent simultaneous installations(when first task failed on timeout)
install_lock = SystemWideLock("Global\\hdp_msi_lock")
try:
# try to acquire lock
if not install_lock.lock():
Logger.info(
"Some other task currently installing hdp.msi, waiting for 10 min for finish"
)
if not install_lock.lock(600000):
raise Fail("Timeout on acquiring lock")
if _validate_msi_install():
Logger.info("hdp.msi already installed")
return
hdp_stack_version = format_hdp_stack_version(stack_version)
hdp_22_specific_props = ''
if hdp_stack_version != "" and compare_versions(
hdp_stack_version, '2.2') >= 0:
hdp_22_specific_props = hdp_22.format(hdp_data_dir=hdp_data_dir)
# install msi
try:
download_file(msi_url, os.path.join(msi_save_dir, save_file))
except:
raise Fail("Failed to download {url}".format(url=msi_url))
File(os.path.join(msi_save_dir, "properties.txt"),
content=cluster_properties.format(
hdp_log_dir=hdp_log_dir,
hdp_data_dir=hdp_data_dir,
local_host=local_host,
db_flavor=db_flavor,
hdp_22_specific_props=hdp_22_specific_props))
hdp_msi_path = os_utils.quote_path(os.path.join(save_dir, "hdp.msi"))
hdp_log_path = os_utils.quote_path(os.path.join(save_dir, "hdp.log"))
hdp_layout_path = os_utils.quote_path(
os.path.join(save_dir, "properties.txt"))
hadoop_password_arg = os_utils.quote_path(hadoop_password)
Execute(
INSTALL_MSI_CMD.format(hdp_msi_path=hdp_msi_path,
hdp_log_path=hdp_log_path,
hdp_layout_path=hdp_layout_path,
hadoop_user=hadoop_user,
hadoop_password_arg=hadoop_password_arg))
reload_windows_env()
# create additional services manually due to hdp.msi limitaitons
_ensure_services_created(hadoop_user, hadoop_password)
_create_symlinks(stack_version)
# finalizing install
_write_marker()
_validate_msi_install()
finally:
install_lock.unlock()
def install_windows_msi(url_base, save_dir, save_files, hadoop_user, hadoop_password, stack_version):
global _working_dir
_working_dir = save_dir
save_dir = os.path.abspath(save_dir)
msi_save_dir = save_dir
# system wide lock to prevent simultaneous installations(when first task failed on timeout)
install_lock = SystemWideLock("Global\\hdp_msi_lock")
try:
# try to acquire lock
if not install_lock.lock():
Logger.info("Some other task currently installing hdp.msi, waiting for 10 min for finish")
if not install_lock.lock(600000):
raise Fail("Timeout on acquiring lock")
if _validate_msi_install():
Logger.info("hdp.msi already installed")
return
stack_version_formatted = format_stack_version(stack_version)
hdp_22_specific_props = ''
if stack_version_formatted != "" and compare_versions(stack_version_formatted, '2.2') >= 0:
hdp_22_specific_props = hdp_22.format(data_dir=data_dir)
# MSIs cannot be larger than 2GB. HDPWIN 2.3 needed split in order to accommodate this limitation
msi_file = ''
for save_file in save_files:
if save_file.lower().endswith(".msi"):
msi_file = save_file
file_url = urlparse.urljoin(url_base, save_file)
try:
download_file(file_url, os.path.join(msi_save_dir, save_file))
except:
raise Fail("Failed to download {url}".format(url=file_url))
File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format(log_dir=log_dir,
data_dir=data_dir,
local_host=local_host,
db_flavor=db_flavor,
hdp_22_specific_props=hdp_22_specific_props))
# install msi
msi_path = os_utils.quote_path(os.path.join(save_dir, msi_file))
log_path = os_utils.quote_path(os.path.join(save_dir, msi_file[:-3] + "log"))
layout_path = os_utils.quote_path(os.path.join(save_dir, "properties.txt"))
hadoop_password_arg = os_utils.quote_path(hadoop_password)
Execute(
INSTALL_MSI_CMD.format(msi_path=msi_path, log_path=log_path, layout_path=layout_path,
hadoop_user=hadoop_user, hadoop_password_arg=hadoop_password_arg))
reload_windows_env()
# create additional services manually due to hdp.msi limitaitons
_ensure_services_created(hadoop_user, hadoop_password)
_create_symlinks(stack_version)
# finalizing install
_write_marker()
_validate_msi_install()
finally:
install_lock.unlock()
def _install_lzo_support_if_needed(self, params):
hadoop_classpath_prefix = self._expand_hadoop_classpath_prefix(params.hadoop_classpath_prefix_template, params.config['configurations']['tez-site'])
hadoop_lzo_dest_path = extract_path_component(hadoop_classpath_prefix, "hadoop-lzo-")
if hadoop_lzo_dest_path:
hadoop_lzo_file = os.path.split(hadoop_lzo_dest_path)[1]
config = Script.get_config()
file_url = urlparse.urljoin(config['hostLevelParams']['jdk_location'], hadoop_lzo_file)
hadoop_lzo_dl_path = os.path.join(config["hostLevelParams"]["agentCacheDir"], hadoop_lzo_file)
download_file(file_url, hadoop_lzo_dl_path)
#This is for protection against configuration changes. It will infect every new destination with the lzo jar,
# but since the classpath points to the jar directly we're getting away with it.
if not os.path.exists(hadoop_lzo_dest_path):
copy_file(hadoop_lzo_dl_path, hadoop_lzo_dest_path)
def setup_jdk():
import params
if not params.jdk_name:
return
if _check_installed():
return
if not os.path.exists(params.java_home):
os.makedirs(params.java_home)
jdk_setup_savepath = os.path.join(params.java_home, params.jdk_name)
jdk_download_url = "{0}/{1}".format(params.jdk_location, params.jdk_name)
download_file(jdk_download_url, jdk_setup_savepath)
Execute(_install_cmd.format(jdk_setup_savepath, params.java_home))
if not _check_installed():
raise Fail("Error when installing jdk")
def _install_lzo_support_if_needed(self, params):
hadoop_classpath_prefix = self._expand_hadoop_classpath_prefix(params.hadoop_classpath_prefix_template, params.config['configurations']['tez-site'])
hadoop_lzo_dest_path = extract_path_component(hadoop_classpath_prefix, "hadoop-lzo-")
if hadoop_lzo_dest_path:
hadoop_lzo_file = os.path.split(hadoop_lzo_dest_path)[1]
config = Script.get_config()
file_url = urlparse.urljoin(config['hostLevelParams']['jdk_location'], hadoop_lzo_file)
hadoop_lzo_dl_path = os.path.join(config["hostLevelParams"]["agentCacheDir"], hadoop_lzo_file)
download_file(file_url, hadoop_lzo_dl_path)
#This is for protection against configuration changes. It will infect every new destination with the lzo jar,
# but since the classpath points to the jar directly we're getting away with it.
if not os.path.exists(hadoop_lzo_dest_path):
copy_file(hadoop_lzo_dl_path, hadoop_lzo_dest_path)
def setup_jdk():
import params
if not params.jdk_name:
return
if _check_installed():
return
if not os.path.exists(params.java_home):
os.makedirs(params.java_home)
jdk_setup_savepath = os.path.join(params.java_home, params.jdk_name)
jdk_download_url = "{0}/{1}".format(params.jdk_location, params.jdk_name)
download_file(jdk_download_url, jdk_setup_savepath)
Execute(_install_cmd.format(jdk_setup_savepath, params.java_home))
if not _check_installed():
raise Fail("Error when installing jdk")
def ensure_jdbc_driver_is_in_classpath(dest_dir, cache_location, driver_url, driver_files):
#Attempt to find the JDBC driver installed locally
#If not, attempt to download it from the server resources URL
for driver_file in driver_files:
dest_path = os.path.join(dest_dir, driver_file)
Logger.info("JDBC driver file(s) {0}: Attempting to copy from {1} or download from {2} to {3}".format(
str(driver_files), cache_location, driver_url, dest_dir))
if not os.path.exists(dest_path):
search_path = os.environ["PATH"]
if cache_location:
search_path += os.pathsep + cache_location #The locally installed version takes precedence over the cache
local_path = search_file(driver_file, search_path)
if not local_path:
download_file(driver_url + "/" + driver_file, dest_path)
else:
copy_file(local_path, dest_path)
def install_windows_msi(msi_url, save_dir, save_file, hadoop_password, stack_version):
global _working_dir
_working_dir = save_dir
save_dir = os.path.abspath(save_dir)
msi_save_dir = save_dir
# system wide lock to prevent simultaneous installations(when first task failed on timeout)
install_lock = SystemWideLock("hdp_msi_lock")
try:
# try to acquire lock
if not install_lock.lock():
Logger.info("Some other task currently installing hdp.msi, waiting for 10 min for finish")
if not install_lock.lock(600000):
raise Fail("Timeout on acquiring lock")
if _validate_msi_install():
Logger.info("hdp.msi already installed")
return
hdp_stack_version = format_hdp_stack_version(stack_version)
hdp_22_specific_props = ''
if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
hdp_22_specific_props = hdp_22.format(hdp_data_dir=hdp_data_dir)
# install msi
download_file(msi_url, os.path.join(msi_save_dir, save_file))
File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format(hdp_log_dir=hdp_log_dir,
hdp_data_dir=hdp_data_dir,
local_host=local_host,
db_flavor=db_flavor,
hdp_22_specific_props=hdp_22_specific_props))
hdp_msi_path = os_utils.quote_path(os.path.join(save_dir, "hdp.msi"))
hdp_log_path = os_utils.quote_path(os.path.join(save_dir, "hdp.log"))
hdp_layout_path = os_utils.quote_path(os.path.join(save_dir, "properties.txt"))
hadoop_password_arg = os_utils.quote_path(hadoop_password)
Execute(
INSTALL_MSI_CMD.format(hdp_msi_path=hdp_msi_path, hdp_log_path=hdp_log_path, hdp_layout_path=hdp_layout_path,
hadoop_password_arg=hadoop_password_arg))
reload_windows_env()
# create additional services manually due to hdp.msi limitaitons
_ensure_services_created(hadoop_password)
_create_symlinks(stack_version)
# finalizing install
_write_marker()
_validate_msi_install()
finally:
install_lock.unlock()
def ensure_jdbc_driver_is_in_classpath(dest_dir, cache_location, driver_url,
driver_files):
#Attempt to find the JDBC driver installed locally
#If not, attempt to download it from the server resources URL
for driver_file in driver_files:
dest_path = os.path.join(dest_dir, driver_file)
Logger.info(
"JDBC driver file(s) {0}: Attempting to copy from {1} or download from {2} to {3}"
.format(str(driver_files), cache_location, driver_url, dest_dir))
if not os.path.exists(dest_path):
search_path = os.environ["PATH"]
if cache_location:
search_path += os.pathsep + cache_location #The locally installed version takes precedence over the cache
local_path = search_file(driver_file, search_path)
if not local_path:
download_file(driver_url + "/" + driver_file, dest_path)
else:
copy_file(local_path, dest_path)
def oozie(is_server=False):
import params
from status_params import oozie_server_win_service_name
XmlConfig(
"oozie-site.xml",
conf_dir=params.oozie_conf_dir,
configurations=params.config['configurations']['oozie-site'],
owner=params.oozie_user,
mode='f',
configuration_attributes=params.config['configuration_attributes']
['oozie-site'])
File(os.path.join(params.oozie_conf_dir, "oozie-env.cmd"),
owner=params.oozie_user,
content=InlineTemplate(params.oozie_env_cmd_template))
Directory(
params.oozie_tmp_dir,
owner=params.oozie_user,
recursive=True,
)
if is_server:
# Manually overriding service logon user & password set by the installation package
ServiceConfig(oozie_server_win_service_name,
action="change_user",
username=params.oozie_user,
password=Script.get_password(params.oozie_user))
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"),
os.path.join(params.oozie_root, "extra_libs", "sqljdbc4.jar"))
webapps_sqljdbc_path = os.path.join(params.oozie_home, "oozie-server",
"webapps", "oozie", "WEB-INF", "lib",
"sqljdbc4.jar")
if os.path.isfile(webapps_sqljdbc_path):
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"), webapps_sqljdbc_path)
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"),
os.path.join(params.oozie_home, "share", "lib", "oozie",
"sqljdbc4.jar"))
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"),
os.path.join(params.oozie_home, "temp", "WEB-INF", "lib",
"sqljdbc4.jar"))
def download_mpack(mpack_path):
"""
Download management pack
:param mpack_path: Path to management pack
:return: Path where the management pack was downloaded
"""
# Download management pack to a temp location
tmpdir = _get_temp_dir()
archive_filename = os.path.basename(mpack_path)
tmp_archive_path = os.path.join(tmpdir, archive_filename)
print_info_msg("Download management pack to temp location {0}".format(tmp_archive_path))
if os.path.exists(tmp_archive_path):
os.remove(tmp_archive_path)
if os.path.exists(mpack_path):
# local path
copy_file(mpack_path, tmp_archive_path)
else:
# remote path
download_file(mpack_path, tmp_archive_path)
return tmp_archive_path
def getHiveMetastorePassword():
passwd = ''
if 'hadoop.security.credential.provider.path' in config['configurations'][
'hive-site']:
# Try to download CredentialUtil.jar from ambari-server resources
cs_lib_path = config['configurations']['hive-site'][
'credentialStoreClassPath']
credential_util_dir = cs_lib_path.split('*')[
0] # Remove the trailing '*'
credential_util_path = os.path.join(credential_util_dir,
credential_util_jar)
credential_util_url = jdk_location + credential_util_jar
try:
download_file(credential_util_url, credential_util_path)
except Exception, e:
message = 'Error downloading {0} from Ambari Server resources. {1}'.format(
credential_util_url, str(e))
Logger.error(message)
raise
# Execute a get command on the CredentialUtil CLI to get the password for the specified alias
java_home = config['hostLevelParams']['java_home']
java_bin = '{java_home}/bin/java'.format(java_home=java_home)
alias = 'javax.jdo.option.ConnectionPassword'
provider_path = config['configurations']['hive-site'][
'hadoop.security.credential.provider.path']
cmd = (java_bin, '-cp', cs_lib_path, credential_util_cmd, 'get', alias,
'-provider', provider_path)
cmd_result, std_out_msg = checked_call(cmd)
if cmd_result != 0:
message = 'The following error occurred while executing {0}: {1}'.format(
' '.join(cmd), std_out_msg)
Logger.error(message)
raise
std_out_lines = std_out_msg.split('\n')
passwd = std_out_lines[
-1] # Get the last line of the output, to skip warnings if any.
def action_install(self):
name = self.resource.msi_name
msi_file_path = name
dict_args = self.resource.dict_args
list_args = self.resource.list_args
working_dir = os.path.abspath(
Script.get_config()["agentLevelParams"]["agentCacheDir"])
http_source = self.resource.http_source
# name can be a path to file in local file system
msi_filename = os.path.split(name)[1]
log_file_path = os.path.join(working_dir, msi_filename) + ".log"
marker_file = os.path.join(working_dir, msi_filename) + ".installed"
# build string from passed arguments to Msi resource
dict_args_str = ' ALLUSERS="1"'
for k, v in dict_args.iteritems():
dict_args_str += " " + str(k) + "=" + str(v)
list_args_str = ''
for a in list_args:
list_args_str += " /" + str(a)
# if http source present we download msi and then execute it
if http_source:
download_url = urlparse.urljoin(http_source, name)
msi_file_path = os.path.join(working_dir, msi_filename)
download_file(download_url, msi_file_path)
if not os.path.exists(marker_file):
Execute(
MsiProvider.MSI_INSTALL_COMMAND.format(
msi_file_path=msi_file_path,
log_file_path=log_file_path,
dict_args_str=dict_args_str,
list_args_str=list_args_str).rstrip())
# writing marker file to not install new msi later
open(marker_file, "w").close()
def oozie(is_server=False):
import params
from status_params import oozie_server_win_service_name
XmlConfig("oozie-site.xml",
conf_dir=params.oozie_conf_dir,
configurations=params.config['configurations']['oozie-site'],
owner=params.oozie_user,
mode='f',
configuration_attributes=params.config['configuration_attributes']['oozie-site']
)
File(os.path.join(params.oozie_conf_dir, "oozie-env.cmd"),
owner=params.oozie_user,
content=InlineTemplate(params.oozie_env_cmd_template)
)
Directory(params.oozie_tmp_dir,
owner=params.oozie_user,
recursive = True,
)
if is_server:
# Manually overriding service logon user & password set by the installation package
ServiceConfig(oozie_server_win_service_name,
action="change_user",
username = params.oozie_user,
password = Script.get_password(params.oozie_user))
download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"),
os.path.join(params.oozie_root, "extra_libs", "sqljdbc4.jar")
)
webapps_sqljdbc_path = os.path.join(params.oozie_home, "oozie-server", "webapps", "oozie", "WEB-INF", "lib", "sqljdbc4.jar")
if os.path.isfile(webapps_sqljdbc_path):
download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"),
webapps_sqljdbc_path
)
download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"),
os.path.join(params.oozie_home, "share", "lib", "oozie", "sqljdbc4.jar")
)
download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"),
os.path.join(params.oozie_home, "temp", "WEB-INF", "lib", "sqljdbc4.jar")
)
def oozie(is_server=False):
import params
XmlConfig(
"oozie-site.xml",
conf_dir=params.oozie_conf_dir,
configurations=params.config['configurations']['oozie-site'],
owner=params.oozie_user,
mode='f',
configuration_attributes=params.config['configuration_attributes']
['oozie-site'])
File(os.path.join(params.oozie_conf_dir, "oozie-env.cmd"),
owner=params.oozie_user,
content=InlineTemplate(params.oozie_env_cmd_template))
Directory(
params.oozie_tmp_dir,
owner=params.oozie_user,
recursive=True,
)
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"),
os.path.join(params.oozie_root, "extra_libs", "sqljdbc4.jar"))
webapps_sqljdbc_path = os.path.join(params.oozie_home, "oozie-server",
"webapps", "oozie", "WEB-INF", "lib",
"sqljdbc4.jar")
if os.path.isfile(webapps_sqljdbc_path):
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"), webapps_sqljdbc_path)
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"),
os.path.join(params.oozie_home, "share", "lib", "oozie",
"sqljdbc4.jar"))
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"),
os.path.join(params.oozie_home, "temp", "WEB-INF", "lib",
"sqljdbc4.jar"))
def execute_db_connection_check(self, config, tmp_dir):
print "DB connection check started."
# initialize needed data
ambari_server_hostname = config['commandParams']['ambari_server_host']
check_db_connection_jar_name = "DBConnectionVerification.jar"
jdk_location = config['commandParams']['jdk_location']
java_home = config['commandParams']['java_home']
db_name = config['commandParams']['db_name']
if db_name == DB_MYSQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MYSQL
jdbc_driver_class = JDBC_DRIVER_CLASS_MYSQL
jdbc_name = JDBC_DRIVER_SYMLINK_MYSQL
elif db_name == DB_ORACLE:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_ORACLE
jdbc_driver_class = JDBC_DRIVER_CLASS_ORACLE
jdbc_name = JDBC_DRIVER_SYMLINK_ORACLE
elif db_name == DB_POSTGRESQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_POSTGRESQL
jdbc_driver_class = JDBC_DRIVER_CLASS_POSTGRESQL
jdbc_name = JDBC_DRIVER_SYMLINK_POSTGRESQL
elif db_name == DB_MSSQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MSSQL
jdbc_driver_class = JDBC_DRIVER_CLASS_MSSQL
jdbc_name = JDBC_DRIVER_SYMLINK_MSSQL
elif db_name == DB_SQLA:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_SQLA
jdbc_driver_class = JDBC_DRIVER_CLASS_SQLA
jdbc_name = JDBC_DRIVER_SYMLINK_SQLA
db_connection_url = config['commandParams']['db_connection_url']
user_name = config['commandParams']['user_name']
user_passwd = config['commandParams']['user_passwd']
agent_cache_dir = os.path.abspath(config["hostLevelParams"]["agentCacheDir"])
check_db_connection_url = jdk_location + check_db_connection_jar_name
jdbc_path = os.path.join(agent_cache_dir, jdbc_name)
class_path_delimiter = ":"
if db_name == DB_SQLA:
jdbc_jar_path = agent_cache_dir + JDBC_DRIVER_SQLA_JAR_PATH_IN_ARCHIVE
java_library_path = agent_cache_dir + JARS_PATH_IN_ARCHIVE_SQLA + class_path_delimiter + agent_cache_dir + \
LIBS_PATH_IN_ARCHIVE_SQLA
else:
jdbc_jar_path = jdbc_path
java_library_path = agent_cache_dir
check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name)
java_bin = "java"
if OSCheck.is_windows_family():
java_bin = "java.exe"
class_path_delimiter = ";"
java_exec = os.path.join(java_home, "bin",java_bin)
if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \
or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec):
message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \
"\n"
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
environment = { "no_proxy": format("{ambari_server_hostname}") }
# download and install java if it doesn't exists
if not os.path.isfile(java_exec):
jdk_name = config['commandParams']['jdk_name']
jdk_url = "{0}/{1}".format(jdk_location, jdk_name)
jdk_download_target = os.path.join(agent_cache_dir, jdk_name)
java_dir = os.path.dirname(java_home)
try:
download_file(jdk_url, jdk_download_target)
except Exception, e:
message = "Error downloading JDK from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
if jdk_name.endswith(".exe"):
install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format(
os_utils.quote_path(jdk_download_target), os_utils.quote_path(java_home),
)
install_path = [java_dir]
try:
Execute(install_cmd, path = install_path)
except Exception, e:
message = "Error installing java.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
def execute_db_connection_check(self, config, tmp_dir):
print "DB connection check started."
# initialize needed data
ambari_server_hostname = config['commandParams']['ambari_server_host']
check_db_connection_jar_name = "DBConnectionVerification.jar"
jdk_location = config['commandParams']['jdk_location']
java64_home = config['commandParams']['java_home']
db_name = config['commandParams']['db_name']
if db_name == DB_MYSQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MYSQL
jdbc_driver = JDBC_DRIVER_MYSQL
jdbc_name = JDBC_DRIVER_SYMLINK_MYSQL
elif db_name == DB_ORACLE:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_ORACLE
jdbc_driver = JDBC_DRIVER_ORACLE
jdbc_name = JDBC_DRIVER_SYMLINK_ORACLE
elif db_name == DB_POSTGRESQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_POSTGRESQL
jdbc_driver = JDBC_DRIVER_POSTGRESQL
jdbc_name = JDBC_DRIVER_SYMLINK_POSTGRESQL
elif db_name == DB_MSSQL:
jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MSSQL
jdbc_driver = JDBC_DRIVER_MSSQL
jdbc_name = JDBC_DRIVER_SYMLINK_MSSQL
db_connection_url = config['commandParams']['db_connection_url']
user_name = config['commandParams']['user_name']
user_passwd = config['commandParams']['user_passwd']
agent_cache_dir = os.path.abspath(config["hostLevelParams"]["agentCacheDir"])
check_db_connection_url = jdk_location + check_db_connection_jar_name
jdbc_path = os.path.join(agent_cache_dir, jdbc_name)
check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name)
java_bin = "java"
class_path_delimiter = ":"
if OSCheck.is_windows_family():
java_bin = "java.exe"
class_path_delimiter = ";"
java_exec = os.path.join(java64_home, "bin",java_bin)
if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \
or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec):
message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \
"\n"
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
environment = { "no_proxy": format("{ambari_server_hostname}") }
# download and install java if it doesn't exists
if not os.path.isfile(java_exec):
jdk_name = config['commandParams']['jdk_name']
jdk_url = "{0}/{1}".format(jdk_location, jdk_name)
jdk_download_target = os.path.join(agent_cache_dir, jdk_name)
java_dir = os.path.dirname(java64_home)
try:
download_file(jdk_url, jdk_download_target)
except Exception, e:
message = "Error downloading JDK from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
if jdk_name.endswith(".bin"):
install_cmd = format("mkdir -p {java_dir} ; chmod +x {jdk_download_target}; cd {java_dir} ; echo A | " \
"{jdk_curl_target} -noregister > /dev/null 2>&1")
install_path = ["/bin","/usr/bin/"]
elif jdk_name.endswith(".gz"):
install_cmd = format("mkdir -p {java_dir} ; cd {java_dir} ; tar -xf {jdk_download_target} > /dev/null 2>&1")
install_path = ["/bin","/usr/bin/"]
elif jdk_name.endswith(".exe"):
install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format(
os_utils.quote_path(jdk_download_target), os_utils.quote_path(java64_home),
)
install_path = [java_dir]
try:
Execute(install_cmd, path = install_path)
except Exception, e:
message = "Error installing java.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format(
os_utils.quote_path(jdk_download_target), os_utils.quote_path(java64_home),
)
install_path = [java_dir]
try:
Execute(install_cmd, path = install_path)
except Exception, e:
message = "Error installing java.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
# download DBConnectionVerification.jar from ambari-server resources
try:
download_file(check_db_connection_url, check_db_connection_path)
except Exception, e:
message = "Error downloading DBConnectionVerification.jar from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
# download jdbc driver from ambari-server resources
try:
download_file(jdbc_url, jdbc_path)
if db_name == DB_MSSQL and OSCheck.is_windows_family():
jdbc_auth_path = os.path.join(agent_cache_dir, JDBC_AUTH_SYMLINK_MSSQL)
jdbc_auth_url = jdk_location + JDBC_AUTH_SYMLINK_MSSQL
download_file(jdbc_auth_url, jdbc_auth_path)
def execute_db_connection_check(self, config, tmp_dir):
Logger.info("DB connection check started.")
# initialize needed data
ambari_server_hostname = config['commandParams']['ambari_server_host']
check_db_connection_jar_name = "DBConnectionVerification.jar"
jdk_location = config['commandParams']['jdk_location']
java_home = config['commandParams']['java_home']
db_name = config['commandParams']['db_name']
no_jdbc_error_message = None
if db_name == DB_MYSQL:
jdbc_driver_mysql_name = default(
"/hostLevelParams/custom_mysql_jdbc_name", None)
if not jdbc_driver_mysql_name:
no_jdbc_error_message = "The MySQL JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=mysql --jdbc-driver=/path/to/jdbc_driver'."
else:
jdbc_url = jdk_location + jdbc_driver_mysql_name
jdbc_driver_class = JDBC_DRIVER_CLASS_MYSQL
jdbc_name = jdbc_driver_mysql_name
elif db_name == DB_ORACLE:
jdbc_driver_oracle_name = default(
"/hostLevelParams/custom_oracle_jdbc_name", None)
if not jdbc_driver_oracle_name:
no_jdbc_error_message = "The Oracle JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=oracle --jdbc-driver=/path/to/jdbc_driver'."
else:
jdbc_url = jdk_location + jdbc_driver_oracle_name
jdbc_driver_class = JDBC_DRIVER_CLASS_ORACLE
jdbc_name = jdbc_driver_oracle_name
elif db_name == DB_POSTGRESQL:
jdbc_driver_postgres_name = default(
"/hostLevelParams/custom_postgres_jdbc_name", None)
if not jdbc_driver_postgres_name:
no_jdbc_error_message = "The Postgres JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=postgres --jdbc-driver=/path/to/jdbc_driver'."
else:
jdbc_url = jdk_location + jdbc_driver_postgres_name
jdbc_driver_class = JDBC_DRIVER_CLASS_POSTGRESQL
jdbc_name = jdbc_driver_postgres_name
elif db_name == DB_MSSQL:
jdbc_driver_mssql_name = default(
"/hostLevelParams/custom_mssql_jdbc_name", None)
if not jdbc_driver_mssql_name:
no_jdbc_error_message = "The MSSQL JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=mssql --jdbc-driver=/path/to/jdbc_driver'."
else:
jdbc_url = jdk_location + jdbc_driver_mssql_name
jdbc_driver_class = JDBC_DRIVER_CLASS_MSSQL
jdbc_name = jdbc_driver_mssql_name
elif db_name == DB_SQLA:
jdbc_driver_sqla_name = default(
"/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
if not jdbc_driver_sqla_name:
no_jdbc_error_message = "The SQLAnywhere JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=sqlanywhere --jdbc-driver=/path/to/jdbc_driver'."
else:
jdbc_url = jdk_location + jdbc_driver_sqla_name
jdbc_driver_class = JDBC_DRIVER_CLASS_SQLA
jdbc_name = jdbc_driver_sqla_name
else:
no_jdbc_error_message = format(
"'{db_name}' database type not supported.")
if no_jdbc_error_message:
Logger.warning(no_jdbc_error_message)
db_connection_check_structured_output = {
"exit_code": 1,
"message": no_jdbc_error_message
}
return db_connection_check_structured_output
db_connection_url = config['commandParams']['db_connection_url']
user_name = config['commandParams']['user_name']
user_passwd = config['commandParams']['user_passwd']
agent_cache_dir = os.path.abspath(
config["hostLevelParams"]["agentCacheDir"])
check_db_connection_url = jdk_location + check_db_connection_jar_name
jdbc_path = os.path.join(agent_cache_dir, jdbc_name)
class_path_delimiter = ":"
if db_name == DB_SQLA:
jdbc_jar_path = agent_cache_dir + JDBC_DRIVER_SQLA_JAR_PATH_IN_ARCHIVE
java_library_path = agent_cache_dir + JARS_PATH_IN_ARCHIVE_SQLA + class_path_delimiter + agent_cache_dir + \
LIBS_PATH_IN_ARCHIVE_SQLA
else:
jdbc_jar_path = jdbc_path
java_library_path = agent_cache_dir
check_db_connection_path = os.path.join(agent_cache_dir,
check_db_connection_jar_name)
java_bin = "java"
if OSCheck.is_windows_family():
java_bin = "java.exe"
class_path_delimiter = ";"
java_exec = os.path.join(java_home, "bin", java_bin)
if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \
or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec):
message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \
"\n"
Logger.warning(message)
db_connection_check_structured_output = {
"exit_code": 1,
"message": message
}
return db_connection_check_structured_output
environment = {"no_proxy": format("{ambari_server_hostname}")}
# download and install java if it doesn't exists
if not os.path.isfile(java_exec):
jdk_name = config['commandParams']['jdk_name']
jdk_url = "{0}/{1}".format(jdk_location, jdk_name)
jdk_download_target = os.path.join(agent_cache_dir, jdk_name)
java_dir = os.path.dirname(java_home)
try:
download_file(jdk_url, jdk_download_target)
except Exception, e:
message = "Error downloading JDK from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
Logger.exception(message)
db_connection_check_structured_output = {
"exit_code": 1,
"message": message
}
return db_connection_check_structured_output
if jdk_name.endswith(".exe"):
install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format(
os_utils.quote_path(jdk_download_target),
os_utils.quote_path(java_home),
)
install_path = [java_dir]
try:
Execute(install_cmd, path=install_path)
except Exception, e:
message = "Error installing java.\n" + str(e)
Logger.exception(message)
db_connection_check_structured_output = {
"exit_code": 1,
"message": message
}
return db_connection_check_structured_output
def install_windows_msi(url_base, save_dir, save_files, hadoop_user,
hadoop_password, stack_version):
global _working_dir
_working_dir = save_dir
save_dir = os.path.abspath(save_dir)
msi_save_dir = save_dir
# system wide lock to prevent simultaneous installations(when first task failed on timeout)
install_lock = SystemWideLock("Global\\hdp_msi_lock")
try:
# try to acquire lock
if not install_lock.lock():
Logger.info(
"Some other task currently installing hdp.msi, waiting for 10 min for finish"
)
if not install_lock.lock(600000):
raise Fail("Timeout on acquiring lock")
if _validate_msi_install():
Logger.info("hdp.msi already installed")
return
stack_version_formatted = format_stack_version(stack_version)
hdp_22_specific_props = ''
if stack_version_formatted != "" and compare_versions(
stack_version_formatted, '2.2') >= 0:
hdp_22_specific_props = hdp_22.format(data_dir=data_dir)
# MSIs cannot be larger than 2GB. HDPWIN 2.3 needed split in order to accommodate this limitation
msi_file = ''
for save_file in save_files:
if save_file.lower().endswith(".msi"):
msi_file = save_file
file_url = urlparse.urljoin(url_base, save_file)
try:
download_file(file_url, os.path.join(msi_save_dir, save_file))
except:
raise Fail("Failed to download {url}".format(url=file_url))
File(os.path.join(msi_save_dir, "properties.txt"),
content=cluster_properties.format(
log_dir=log_dir,
data_dir=data_dir,
local_host=local_host,
db_flavor=db_flavor,
hdp_22_specific_props=hdp_22_specific_props))
# install msi
msi_path = os_utils.quote_path(os.path.join(save_dir, msi_file))
log_path = os_utils.quote_path(
os.path.join(save_dir, msi_file[:-3] + "log"))
layout_path = os_utils.quote_path(
os.path.join(save_dir, "properties.txt"))
hadoop_password_arg = os_utils.quote_path(hadoop_password)
Execute(
INSTALL_MSI_CMD.format(msi_path=msi_path,
log_path=log_path,
layout_path=layout_path,
hadoop_user=hadoop_user,
hadoop_password_arg=hadoop_password_arg))
reload_windows_env()
# create additional services manually due to hdp.msi limitaitons
_ensure_services_created(hadoop_user, hadoop_password)
_create_symlinks(stack_version)
# finalizing install
_write_marker()
_validate_msi_install()
finally:
install_lock.unlock()
install_cmd = format("mkdir -p {tmp_java_dir} && cd {tmp_java_dir} && tar -xf {jdk_download_target} && {sudo} cp -rp {tmp_java_dir}/* {java_dir}")
try:
Directory(java_dir)
Execute(chmod_cmd, not_if = format("test -e {java_exec}"), sudo = True)
Execute(install_cmd, not_if = format("test -e {java_exec}"))
File(format("{java_home}/bin/java"), mode=0755, cd_access="a")
Execute(("chown","-R", getpass.getuser(), java_home), sudo = True)
except Exception, e:
message = "Error installing java.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
# download DBConnectionVerification.jar from ambari-server resources
try:
download_file(check_db_connection_url, check_db_connection_path)
except Exception, e:
message = "Error downloading DBConnectionVerification.jar from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
print message
db_connection_check_structured_output = {"exit_code" : 1, "message": message}
return db_connection_check_structured_output
# download jdbc driver from ambari-server resources
try:
download_file(jdbc_url, jdbc_path)
if db_name == DB_MSSQL and OSCheck.is_windows_family():
jdbc_auth_path = os.path.join(agent_cache_dir, JDBC_AUTH_SYMLINK_MSSQL)
jdbc_auth_url = jdk_location + JDBC_AUTH_SYMLINK_MSSQL
download_file(jdbc_auth_url, jdbc_auth_path)