def _install_jdbc_driver(self, properties, files_list):
if type(files_list) is not int:
print 'Copying JDBC drivers to server resources...'
resources_dir = get_resources_location(properties)
db_name = self.dbms_full_name.lower()
symlink_name = db_name + "-jdbc-driver.jar"
jdbc_symlink = os.path.join(resources_dir, symlink_name)
db_default_driver_path = self._get_default_driver_path(properties)
if os.path.lexists(jdbc_symlink):
os.remove(jdbc_symlink)
copy_status = copy_files(files_list, resources_dir)
if not copy_status == 0:
raise FatalException(
-1, "Failed to copy JDBC drivers to server resources")
if db_default_driver_path in files_list:
os.symlink(os.path.join(resources_dir, self.driver_file_name),
jdbc_symlink)
else:
if files_list == -1:
return False
return True
def _cache_jdbc_driver(args):
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
raise FatalException(-1, err)
resources_dir = get_resources_location(properties)
if args.jdbc_driver.endswith(TAR_GZ_ARCHIVE_TYPE):
symlink_name = args.jdbc_db + "-jdbc-driver" + TAR_GZ_ARCHIVE_TYPE
else:
symlink_name = args.jdbc_db + "-jdbc-driver.jar"
jdbc_symlink = os.path.join(resources_dir, symlink_name)
path, jdbc_name = os.path.split(args.jdbc_driver)
if os.path.lexists(jdbc_symlink):
os.remove(jdbc_symlink)
if os.path.isfile(os.path.join(resources_dir, jdbc_name)):
os.remove(os.path.join(resources_dir, jdbc_name))
try:
shutil.copy(args.jdbc_driver, resources_dir)
print "Copying {0} to {1}".format(args.jdbc_driver, resources_dir)
except Exception, e:
err = "Can not copy file {0} to {1} due to: {2} . Please check file " \
"permissions and free disk space.".format(args.jdbc_driver, resources_dir, str(e))
raise FatalException(1, err)
def _cache_jdbc_driver(args):
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
raise FatalException(-1, err)
resources_dir = get_resources_location(properties)
path, jdbc_name = os.path.split(args.jdbc_driver)
custom_db_jdbc_property_name = "custom." + args.jdbc_db + ".jdbc.name"
custom_db_jdbc_property_value = properties[custom_db_jdbc_property_name]
symlink_name = os.path.join(resources_dir, default_connectors_map[args.jdbc_db])
if custom_db_jdbc_property_value:
properties.process_pair("previous." + custom_db_jdbc_property_name, custom_db_jdbc_property_value)
else:
if os.path.lexists(symlink_name):
properties.process_pair("previous." + custom_db_jdbc_property_name, default_connectors_map[args.jdbc_db])
properties.process_pair(custom_db_jdbc_property_name, jdbc_name)
dest = os.path.join(resources_dir, jdbc_name)
if os.path.isfile(dest):
os.remove(dest)
try:
shutil.copy(args.jdbc_driver, dest)
print "Copying {0} to {1}".format(args.jdbc_driver, dest)
except Exception, e:
err = "Cannot copy file {0} to {1} due to: {2} . Please check file " \
"permissions and free disk space.".format(args.jdbc_driver, dest, str(e))
raise FatalException(1, err)
def _extract_client_tarball(self, properties):
files = []
files.extend(
glob.glob(configDefaults.SHARE_PATH + os.sep +
self.client_tarball_pattern))
if len(files) > 1:
raise FatalException(
-1, "More than One SQl Anywhere client tarball detected")
elif len(files) == 0:
raise FatalException(-1, self.JDBC_DRIVER_INSTALL_MSG)
cmd = SQLAConfig.EXTRACT_CMD.format(files[0],
get_resources_location(properties))
process = subprocess.Popen(cmd.split(' '),
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = process.communicate()
retcode = process.returncode
if retcode != 0:
raise FatalException(
-1,
"Error extracting SQL Anywhere client tarball: " + str(err))
def refresh_stack_hash(properties):
resources_location = get_resources_location(properties)
stacks_location = get_stack_location(properties)
resource_files_keeper = ResourceFilesKeeper(resources_location, stacks_location)
try:
print "Organizing resource files at {0}...".format(resources_location,
verbose=get_verbose())
resource_files_keeper.perform_housekeeping()
except KeeperException, ex:
msg = "Can not organize resource files at {0}: {1}".format(
resources_location, str(ex))
raise FatalException(-1, msg)
def refresh_stack_hash(properties):
resources_location = get_resources_location(properties)
stacks_location = get_stack_location(properties)
resource_files_keeper = ResourceFilesKeeper(resources_location, stacks_location)
try:
print "Organizing resource files at {0}...".format(resources_location,
verbose=get_verbose())
resource_files_keeper.perform_housekeeping()
except KeeperException, ex:
msg = "Can not organize resource files at {0}: {1}".format(
resources_location, str(ex))
raise FatalException(-1, msg)
def add_jdbc_properties(properties):
for db_name in CUSTOM_JDBC_DB_NAMES:
if db_name == "sqlanywhere":
symlink_name = db_name + "-jdbc-driver" + TAR_GZ_ARCHIVE_TYPE
else:
symlink_name = db_name + "-jdbc-driver.jar"
resources_dir = get_resources_location(properties)
custom_db_jdbc_property_name = "custom." + db_name + ".jdbc.name"
if os.path.lexists(os.path.join(resources_dir, symlink_name)):
properties.process_pair(custom_db_jdbc_property_name, symlink_name)
properties.process_pair("previous." + custom_db_jdbc_property_name, default_connectors_map[db_name])
update_properties(properties)
def download_and_unpack_jce_policy(self, properties):
err_msg_stdout = "JCE Policy files are required for secure HDP setup. Please ensure " \
" all hosts have the JCE unlimited strength policy 6, files."
resources_dir = get_resources_location(properties)
jdk_cfg = self.jdks[self.jdk_index]
try:
JDKSetup._download_jce_policy(jdk_cfg.jcpol_url, jdk_cfg.dest_jcpol_file, resources_dir, properties)
except FatalException, e:
print err_msg_stdout
print_error_msg("Failed to download JCE policy files:")
if e.reason is not None:
print_error_msg("\nREASON: {0}".format(e.reason))
def main():
properties = get_ambari_properties()
if properties == -1:
print >> sys.stderr, "Error getting ambari properties"
return -1
resources_location = get_resources_location(properties)
views_dir = os.path.join(resources_location, "views")
for file in os.listdir(views_dir):
path = os.path.join(views_dir, file)
if os.path.isfile(path):
if "ambari-admin" in path or "storm-view" in path:
print "Keeping views jar : " + path
else:
print "Deleting views jar : " + path
sudo.unlink(path)
else:
print "Deleting views directory : " + path
sudo.rmtree(path)
return 0
def _cache_jdbc_driver(args):
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
raise FatalException(-1, err)
resources_dir = get_resources_location(properties)
path, jdbc_name = os.path.split(args.jdbc_driver)
properties.process_pair("custom." + args.jdbc_db + ".jdbc.name", jdbc_name)
if os.path.isfile(os.path.join(resources_dir, jdbc_name)):
os.remove(os.path.join(resources_dir, jdbc_name))
try:
shutil.copy(args.jdbc_driver, resources_dir)
print "Copying {0} to {1}".format(args.jdbc_driver, resources_dir)
except Exception, e:
err = "Can not copy file {0} to {1} due to: {2} . Please check file " \
"permissions and free disk space.".format(args.jdbc_driver, resources_dir, str(e))
raise FatalException(1, err)
def setup_jce_policy(args):
logger.info("Setup JCE policy for ambari-server.")
if not os.path.exists(args[1]):
err = "Can not run 'setup-jce'. Invalid path {0}.".format(args[1])
raise FatalException(1, err)
properties = get_ambari_properties()
resources_dir = get_resources_location(properties)
zip_path = os.path.split(args[1])
zip_dir = zip_path[0]
if not zip_dir == resources_dir:
try:
shutil.copy(args[1], resources_dir)
except Exception as e:
err = "Fail while trying to copy {0} to {1}. {2}".format(
args[1], resources_dir, e)
raise FatalException(1, err)
jdk_path = properties.get_property(JAVA_HOME_PROPERTY)
if not jdk_path or not os.path.exists(jdk_path):
err = "JDK not installed, you need to run 'ambari-server setup' before attempting to install the JCE policy."
raise FatalException(1, err)
zip_name = zip_path[1]
properties.process_pair(JCE_NAME_PROPERTY, zip_name)
print 'Installing JCE policy...'
try:
JDKSetup.unpack_jce_policy(jdk_path, resources_dir, zip_name)
except FatalException as e:
err = 'Installing JCE failed: {0}. Exiting.'.format(e)
raise FatalException(e.code, err)
update_properties(properties)
print 'NOTE: Restart Ambari Server to apply changes' + \
' ("ambari-server restart|stop|start")'
def check_jdbc_drivers(args):
# create jdbc symlinks if jdbc drivers are available in resources
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
resources_dir = get_resources_location(properties)
try:
db_idx_orig = args.database_index
except AttributeError:
db_idx_orig = None
factory = DBMSConfigFactory()
# AMBARI-5696 Validate the symlinks for each supported driver, in case various back-end HDP services happen to
# use different DBMSes
# This is skipped on Windows
db_idx = 1
try:
while db_idx < len(factory.get_supported_dbms()):
args.database_index = db_idx
dbms = factory.create(args, properties)
if dbms.driver_symlink_name:
jdbc_file_path = os.path.join(resources_dir,
dbms.driver_file_name)
if os.path.isfile(jdbc_file_path):
jdbc_symlink = os.path.join(resources_dir,
dbms.driver_symlink_name)
if os.path.lexists(jdbc_symlink):
os.remove(jdbc_symlink)
os.symlink(jdbc_file_path, jdbc_symlink)
db_idx += 1
finally:
args.database_index = db_idx_orig
def _extract_client_tarball(self, properties):
files = []
files.extend(glob.glob(configDefaults.SHARE_PATH + os.sep + self.client_tarball_pattern))
if len(files) > 1:
raise FatalException(-1, "More than One SQl Anywhere client tarball detected")
elif len(files) == 0:
raise FatalException(-1, self.JDBC_DRIVER_INSTALL_MSG)
cmd = SQLAConfig.EXTRACT_CMD.format(files[0], get_resources_location(properties))
process = subprocess.Popen(cmd.split(' '),
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE
)
out, err = process.communicate()
retcode = process.returncode
if retcode != 0:
raise FatalException(-1, "Error extracting SQL Anywhere client tarball: " + str(err))
def setup_jce_policy(args):
if not os.path.exists(args[1]):
err = "Can not run 'setup-jce'. Invalid path {0}.".format(args[1])
raise FatalException(1, err)
properties = get_ambari_properties()
resources_dir = get_resources_location(properties)
zip_path = os.path.split(args[1])
zip_dir = zip_path[0]
if not zip_dir == resources_dir:
try:
shutil.copy(args[1], resources_dir)
except Exception as e:
err = "Fail while trying to copy {0} to {1}. {2}".format(args[1], resources_dir, e)
raise FatalException(1, err)
jdk_path = properties.get_property(JAVA_HOME_PROPERTY)
if not jdk_path or not os.path.exists(jdk_path):
err = "JDK not installed, you need to run 'ambari-server setup' before attempting to install the JCE policy."
raise FatalException(1, err)
zip_name = zip_path[1]
properties.process_pair(JCE_NAME_PROPERTY, zip_name)
print 'Installing JCE policy...'
try:
JDKSetup.unpack_jce_policy(jdk_path, resources_dir, zip_name)
except FatalException as e:
err = 'Installing JCE failed: {0}. Exiting.'.format(e)
raise FatalException(e.code, err)
update_properties(properties)
print 'NOTE: Restart Ambari Server to apply changes' + \
' ("ambari-server restart|stop|start")'
def _install_jdbc_driver(self, properties, files_list):
if type(files_list) is not int:
print 'Copying JDBC drivers to server resources...'
resources_dir = get_resources_location(properties)
db_name = self.dbms_full_name.lower()
symlink_name = db_name + "-jdbc-driver.jar"
jdbc_symlink = os.path.join(resources_dir, symlink_name)
db_default_driver_path = self._get_default_driver_path(properties)
if os.path.lexists(jdbc_symlink):
os.remove(jdbc_symlink)
copy_status = copy_files(files_list, resources_dir)
if not copy_status == 0:
raise FatalException(-1, "Failed to copy JDBC drivers to server resources")
if db_default_driver_path in files_list:
os.symlink(os.path.join(resources_dir, self.driver_file_name), jdbc_symlink)
else:
if files_list == -1:
return False
return True
def check_jdbc_drivers(args):
# create jdbc symlinks if jdbc drivers are available in resources
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
resources_dir = get_resources_location(properties)
try:
db_idx_orig = args.database_index
except AttributeError:
db_idx_orig = None
factory = DBMSConfigFactory()
# AMBARI-5696 Validate the symlinks for each supported driver, in case various back-end HDP services happen to
# use different DBMSes
# This is skipped on Windows
db_idx = 1
try:
while db_idx < len(factory.get_supported_dbms()):
args.database_index = db_idx
dbms = factory.create(args, properties)
if dbms.driver_symlink_name:
jdbc_file_path = os.path.join(resources_dir, dbms.driver_file_name)
if os.path.isfile(jdbc_file_path):
jdbc_symlink = os.path.join(resources_dir, dbms.driver_symlink_name)
if os.path.lexists(jdbc_symlink):
os.remove(jdbc_symlink)
os.symlink(jdbc_file_path, jdbc_symlink)
db_idx += 1
finally:
args.database_index = db_idx_orig
def _get_native_libs(self, properties): return os.path.join(get_resources_location(properties), self.client_folder, "native", "lib64")
def download_and_install_jdk(self, args, properties):
conf_file = properties.fileName
jcePolicyWarn = "JCE Policy files are required for configuring Kerberos security. If you plan to use Kerberos," \
"please make sure JCE Unlimited Strength Jurisdiction Policy Files are valid on all hosts."
if args.java_home:
#java_home was specified among the command-line arguments. Use it as custom JDK location.
if not validate_jdk(args.java_home):
err = "Path to java home " + args.java_home + " or java binary file does not exists"
raise FatalException(1, err)
print_warning_msg("JAVA_HOME " + args.java_home + " must be valid on ALL hosts")
print_warning_msg(jcePolicyWarn)
IS_CUSTOM_JDK = True
properties.process_pair(JAVA_HOME_PROPERTY, args.java_home)
properties.removeOldProp(JDK_NAME_PROPERTY)
properties.removeOldProp(JCE_NAME_PROPERTY)
self._ensure_java_home_env_var_is_set(args.java_home)
self.jdk_index = self.custom_jdk_number
return
java_home_var = get_JAVA_HOME()
if OS_FAMILY == OSConst.WINSRV_FAMILY:
progress_func = None
else:
progress_func = download_progress
if get_silent():
if not java_home_var:
#No java_home_var set, detect if java is already installed
if os.environ.has_key(JAVA_HOME):
args.java_home = os.environ[JAVA_HOME]
properties.process_pair(JAVA_HOME_PROPERTY, args.java_home)
properties.removeOldProp(JDK_NAME_PROPERTY)
properties.removeOldProp(JCE_NAME_PROPERTY)
self._ensure_java_home_env_var_is_set(args.java_home)
self.jdk_index = self.custom_jdk_number
return
else:
# For now, changing the existing JDK to make sure we use a supported one
pass
if java_home_var:
change_jdk = get_YN_input("Do you want to change Oracle JDK [y/n] (n)? ", False)
if not change_jdk:
self._ensure_java_home_env_var_is_set(java_home_var)
self.jdk_index = self.custom_jdk_number
return
#Continue with the normal setup, taking the first listed JDK version as the default option
jdk_num = str(self.jdk_index + 1)
(self.jdks, jdk_choice_prompt, jdk_valid_choices, self.custom_jdk_number) = self._populate_jdk_configs(properties, jdk_num)
jdk_num = get_validated_string_input(
jdk_choice_prompt,
jdk_num,
jdk_valid_choices,
"Invalid number.",
False
)
self.jdk_index = int(jdk_num) - 1
if self.jdk_index == self.custom_jdk_number:
print_warning_msg("JDK must be installed on all hosts and JAVA_HOME must be valid on all hosts.")
print_warning_msg(jcePolicyWarn)
args.java_home = get_validated_string_input("Path to JAVA_HOME: ", None, None, None, False, False)
if not os.path.exists(args.java_home) or not os.path.isfile(os.path.join(args.java_home, "bin", self.JAVA_BIN)):
err = "Java home path or java binary file is unavailable. Please put correct path to java home."
raise FatalException(1, err)
print "Validating JDK on Ambari Server...done."
properties.process_pair(JAVA_HOME_PROPERTY, args.java_home)
properties.removeOldProp(JDK_NAME_PROPERTY)
properties.removeOldProp(JCE_NAME_PROPERTY)
# Make sure any previously existing JDK and JCE name properties are removed. These will
# confuse things in a Custom JDK scenario
properties.removeProp(JDK_NAME_PROPERTY)
properties.removeProp(JCE_NAME_PROPERTY)
self._ensure_java_home_env_var_is_set(args.java_home)
return
jdk_cfg = self.jdks[self.jdk_index]
resources_dir = get_resources_location(properties)
dest_file = os.path.abspath(os.path.join(resources_dir, jdk_cfg.dest_file))
if os.path.exists(dest_file):
print "JDK already exists, using " + dest_file
elif properties[JDK_DOWNLOAD_SUPPORTED_PROPERTY].upper() == "FALSE":
print "ERROR: Oracle JDK is not found in {1}. JDK download is not supported in this distribution. Please download Oracle JDK " \
"archive ({0}) manually from Oracle site, place it into {1} and re-run this script.".format(jdk_cfg.dest_file, dest_file)
print "NOTE: If you have already downloaded the file, please verify if the name is exactly same as {0}.".format(jdk_cfg.dest_file)
print 'Exiting...'
sys.exit(1)
else:
ok = get_YN_input("To download the Oracle JDK and the Java Cryptography Extension (JCE) "
"Policy Files you must accept the "
"license terms found at "
"http://www.oracle.com/technetwork/java/javase/"
"terms/license/index.html and not accepting will "
"cancel the Ambari Server setup and you must install the JDK and JCE "
"files manually.\nDo you accept the "
"Oracle Binary Code License Agreement [y/n] (y)? ", True)
if not ok:
print 'Exiting...'
sys.exit(1)
jdk_url = jdk_cfg.url
print 'Downloading JDK from ' + jdk_url + ' to ' + dest_file
self._download_jdk(jdk_url, dest_file, progress_func)
try:
(retcode, out, java_home_dir) = self._install_jdk(dest_file, jdk_cfg)
except Exception, e:
print "Installation of JDK has failed: %s\n" % str(e)
file_exists = os.path.isfile(dest_file)
if file_exists:
ok = get_YN_input("JDK found at " + dest_file + ". "
"Would you like to re-download the JDK [y/n] (y)? ", not get_silent())
if not ok:
err = "Unable to install JDK. Please remove JDK file found at " + \
dest_file + " and re-run Ambari Server setup"
raise FatalException(1, err)
else:
jdk_url = jdk_cfg.url
print 'Re-downloading JDK from ' + jdk_url + ' to ' + dest_file
self._download_jdk(jdk_url, dest_file, progress_func)
print 'Successfully re-downloaded JDK distribution to ' + dest_file
try:
(retcode, out) = self._install_jdk(dest_file, jdk_cfg)
except Exception, e:
print "Installation of JDK was failed: %s\n" % str(e)
err = "Unable to install JDK. Please remove JDK, file found at " + \
dest_file + " and re-run Ambari Server setup"
raise FatalException(1, err)
def _get_default_driver_path(self, properties): return os.path.join(get_resources_location(properties), self.client_folder, "java", self.driver_file_name)
def _get_native_libs(self, properties): return os.path.join(get_resources_location(properties), self.client_folder, "native", "lib64")
pass
# Add master key and credential store if exists
keyLocation = get_master_key_location(properties)
masterKeyFile = search_file(SECURITY_MASTER_KEY_FILENAME, keyLocation)
if masterKeyFile:
configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((masterKeyFile, configDefaults.MASTER_KEY_FILE_PERMISSIONS, "{0}", False))
credStoreFile = get_credential_store_location(properties)
if os.path.exists(credStoreFile):
configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((credStoreFile, configDefaults.CREDENTIALS_STORE_FILE_PERMISSIONS, "{0}", False))
trust_store_location = properties[SSL_TRUSTSTORE_PATH_PROPERTY]
if trust_store_location:
configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((trust_store_location, configDefaults.TRUST_STORE_LOCATION_PERMISSIONS, "{0}", False))
# Update JDK and JCE permissions
resources_dir = get_resources_location(properties)
jdk_file_name = properties.get_property(JDK_NAME_PROPERTY)
jce_file_name = properties.get_property(JCE_NAME_PROPERTY)
java_home = properties.get_property(JAVA_HOME_PROPERTY)
if jdk_file_name:
jdk_file_path = os.path.abspath(os.path.join(resources_dir, jdk_file_name))
if(os.path.exists(jdk_file_path)):
configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((jdk_file_path, "644", "{0}", False))
if jce_file_name:
jce_file_path = os.path.abspath(os.path.join(resources_dir, jce_file_name))
if(os.path.exists(jce_file_path)):
configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((jce_file_path, "644", "{0}", False))
if java_home:
jdk_security_dir = os.path.abspath(os.path.join(java_home, configDefaults.JDK_SECURITY_DIR))
if(os.path.exists(jdk_security_dir)):
configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((jdk_security_dir, "644", "{0}", True))
def _get_default_driver_path(self, properties): return os.path.join(get_resources_location(properties), self.client_folder, "java", self.driver_file_name)