def check_selinux():
try:
retcode, out, err = run_os_command(GET_SE_LINUX_ST_CMD)
se_status = re.search('(disabled|enabled)', out).group(0)
print "SELinux status is '" + se_status + "'"
if se_status == SE_STATUS_DISABLED:
return 0
else:
try:
se_mode = re.search('(enforcing|permissive)', out).group(0)
except AttributeError:
err = "Error determining SELinux mode. Exiting."
raise FatalException(1, err)
print "SELinux mode is '" + se_mode + "'"
if se_mode == SE_MODE_ENFORCING:
print "Temporarily disabling SELinux"
run_os_command(SE_SETENFORCE_CMD)
print_warning_msg(
"SELinux is set to 'permissive' mode and temporarily disabled.")
ok = get_YN_input("OK to continue [y/n] (y)? ", True)
if not ok:
raise FatalException(1, None)
return 0
except OSError:
print_warning_msg("Could not run {0}: OK".format(GET_SE_LINUX_ST_CMD))
return 0
def remove_file(filePath):
if os.path.exists(filePath):
try:
os.remove(filePath)
except Exception, e:
print_warning_msg('Unable to remove file: ' + str(e))
return 1
def update_krb_jaas_login_properties():
"""
Update configuration files
:return: int -2 - skipped, -1 - error, 0 - successful
"""
prev_conf_file = search_file(configDefaults.AMBARI_KRB_JAAS_LOGIN_BACKUP_FILE, get_conf_dir())
conf_file = search_file(AMBARI_KRB_JAAS_LOGIN_FILE, get_conf_dir())
# check if source and target files exists, if not - skip copy action
if prev_conf_file is None or conf_file is None:
return -2
# if rpmsave file contains invalid keytab, we can skip restoring
if not is_jaas_keytab_exists(prev_conf_file):
return -2
try:
# restore original file, destination arg for rename func shouldn't exists
os.remove(conf_file)
os.rename(prev_conf_file, conf_file)
print_warning_msg("Original file %s kept" % AMBARI_KRB_JAAS_LOGIN_FILE)
except OSError as e:
print "Couldn't move %s file: %s" % (prev_conf_file, e)
return -1
return 0
def _create_custom_user(self):
user = get_validated_string_input(
"Enter user account for ambari-server daemon (root):",
self.user,
"^[a-z_][a-z0-9_-]{1,31}$",
"Invalid username.",
False
)
print_info_msg("Trying to create user {0}".format(user))
command = self.NR_USERADD_CMD.format(user, self.NR_USER_COMMENT)
retcode, out, err = run_os_command(command)
if retcode == 9: # 9 = username already in use
print_info_msg("User {0} already exists, "
"skipping user creation".format(user))
elif retcode != 0: # fail
print_warning_msg("Can't create user {0}. Command {1} "
"finished with {2}: \n{3}".format(user, command, retcode, err))
return retcode
print_info_msg("User configuration is done.")
self.user = user
return 0
def make_process_independent():
processId = os.getpid()
if processId > 0:
try:
os.setpgid(processId, processId)
except OSError, e:
print_warning_msg('setpgid({0}, {0}) failed - {1}'.format(pidJava, str(e)))
pass
def os_change_owner(filePath, user, recursive):
if recursive:
params = " -R "
else:
params = ""
command = NR_CHOWN_CMD.format(params, user, filePath)
retcode, out, err = os_run_os_command(command)
if retcode != 0:
print_warning_msg(WARN_MSG.format(command, filePath, err))
def _setup_remote_db(self):
setup_msg = "Before starting Ambari Server, you must run the following DDL " \
"against the database to create the schema: {0}".format(self.init_script_file)
print_warning_msg(setup_msg)
proceed = get_YN_input("Proceed with configuring remote database connection properties [y/n] (y)? ", True)
retCode = 0 if proceed else -1
return retCode
def _reset_remote_database(self):
client_usage_cmd_drop = self._get_remote_script_line(self.drop_tables_script_file)
client_usage_cmd_init = self._get_remote_script_line(self.init_script_file)
print_warning_msg('To reset Ambari Server schema ' +
'you must run the following DDL against the database to '
+ 'drop the schema:' + os.linesep + client_usage_cmd_drop
+ os.linesep + 'Then you must run the following DDL ' +
'against the database to create the schema: ' + os.linesep +
client_usage_cmd_init + os.linesep)
def _prompt_jdbc_driver_install(self, properties):
result = self._is_jdbc_driver_installed(properties)
if result == -1:
if get_silent():
print_error_msg(self.JDBC_DRIVER_INSTALL_MSG)
else:
print_warning_msg(self.JDBC_DRIVER_INSTALL_MSG)
raw_input(PRESS_ENTER_MSG)
result = self._is_jdbc_driver_installed(properties)
return (result, self.JDBC_DRIVER_INSTALL_MSG)
def os_set_file_permissions(file, mod, recursive, user):
if recursive:
params = " -R "
else:
params = ""
command = NR_CHMOD_CMD.format(params, mod, file)
retcode, out, err = os_run_os_command(command)
if retcode != 0:
print_warning_msg(WARN_MSG.format(command, file, err))
os_change_owner(file, user, recursive)
def make_process_independent():
if IS_FOREGROUND: # upstart script is not able to track process from different pgid.
return
processId = os.getpid()
if processId > 0:
try:
os.setpgid(processId, processId)
except OSError, e:
print_warning_msg('setpgid({0}, {0}) failed - {1}'.format(pidJava, str(e)))
pass
def remove_password_file(filename):
conf_file = find_properties_file()
passFilePath = os.path.join(os.path.dirname(conf_file),
filename)
if os.path.exists(passFilePath):
try:
os.remove(passFilePath)
except Exception, e:
print_warning_msg('Unable to remove password file: ' + str(e))
return 1
def _ensure_java_home_env_var_is_set(self, java_home_dir):
if not os.environ.has_key(JAVA_HOME) or os.environ[JAVA_HOME] != java_home_dir:
java_home_dir_unesc = compress_backslashes(java_home_dir)
retcode, out, err = run_os_command("SETX {0} {1} /M".format(JAVA_HOME, java_home_dir_unesc))
if retcode != 0:
print_warning_msg("SETX output: " + out)
print_warning_msg("SETX error output: " + err)
err = "Setting JAVA_HOME failed. Exit code={0}".format(retcode)
raise FatalException(1, err)
os.environ[JAVA_HOME] = java_home_dir
def check_firewall():
firewall_obj = Firewall().getFirewallObject()
firewall_on = firewall_obj.check_firewall()
if firewall_obj.stderrdata and len(firewall_obj.stderrdata) > 0:
print firewall_obj.stderrdata
if firewall_on:
print_warning_msg("%s is running. Confirm the necessary Ambari ports are accessible. " %
firewall_obj.FIREWALL_SERVICE_NAME +
"Refer to the Ambari documentation for more details on ports.")
ok = get_YN_input("OK to continue [y/n] (y)? ", True)
if not ok:
raise FatalException(1, None)
def exec_ams_env_cmd(options):
ams_env_cmd = os.path.join(options.conf_dir, AMS_ENV_CMD)
if os.path.exists(ams_env_cmd):
cmds = ["cmd.exe", "/C", ams_env_cmd]
procAms = subprocess.Popen(cmds, env=os.environ)
out, err = procAms.communicate()
if err is not None and err is not "":
print_warning_msg(AMS_ENV_CMD + " error output: " + err)
if out is not None and out is not "":
print_info_msg(AMS_ENV_CMD + " output: " + out)
else:
err = 'ERROR: Cannot execute "{0}"'.format(ams_env_cmd)
raise FatalException(1, err)
def os_set_file_permissions(file, mod, recursive, user):
WARN_MSG = "Command {0} returned exit code {1} with message: {2}"
if recursive:
params = " -R "
else:
params = ""
command = NR_CHMOD_CMD.format(params, mod, file)
retcode, out, err = os_run_os_command(command)
if retcode != 0:
print_warning_msg(WARN_MSG.format(command, file, err))
command = NR_CHOWN_CMD.format(params, user, file)
retcode, out, err = os_run_os_command(command)
if retcode != 0:
print_warning_msg(WARN_MSG.format(command, file, err))
def win_main():
parser = init_options_parser()
(options, args) = parser.parse_args()
options.warnings = []
options.exit_message = None
if options.debug:
sys.frozen = 'windows_exe' # Fake py2exe so we can debug
if len(args) == 0:
print parser.print_help()
parser.error("No action entered")
action = args[0]
try:
if action == SETUP_ACTION:
svcsetup()
elif action == START_ACTION:
start(options)
elif action == STOP_ACTION:
stop()
elif action == RESTART_ACTION:
stop()
start(options)
elif action == STATUS_ACTION:
svcstatus(options)
else:
parser.error("Invalid action")
if options.warnings:
for warning in options.warnings:
print_warning_msg(warning)
pass
options.exit_message = "Ambari Metrics Host Monitoring '%s' completed with warnings." % action
pass
except FatalException as e:
if e.reason is not None:
print_error_msg("Exiting with exit code {0}. \nREASON: {1}".format(e.code, e.reason))
sys.exit(e.code)
except NonFatalException as e:
options.exit_message = "Ambari Metrics Host Monitoring '%s' completed with warnings." % action
if e.reason is not None:
print_warning_msg(e.reason)
if options.exit_message is not None:
print options.exit_message
sys.exit(0)
def is_valid_cert_exp(certInfoDict):
if certInfoDict.has_key(NOT_BEFORE_ATTR):
notBefore = certInfoDict[NOT_BEFORE_ATTR]
else:
print_warning_msg('There is no Not Before value in Certificate')
return False
if certInfoDict.has_key(NOT_AFTER_ATTR):
notAfter = certInfoDict['notAfter']
else:
print_warning_msg('There is no Not After value in Certificate')
return False
notBeforeDate = datetime.datetime.strptime(notBefore, SSL_DATE_FORMAT)
notAfterDate = datetime.datetime.strptime(notAfter, SSL_DATE_FORMAT)
currentDate = datetime.datetime.now()
if currentDate > notAfterDate:
print_warning_msg('Certificate expired on: ' + str(notAfterDate))
return False
if currentDate < notBeforeDate:
print_warning_msg('Certificate will be active from: ' + str(notBeforeDate))
return False
return True
def main():
from amc_service import init_options_parser, init_service_debug, setup, start, stop, svcstatus
parser = init_options_parser()
(options, args) = parser.parse_args()
options.warnings = []
options.exit_message = None
init_service_debug(options)
if len(args) == 0:
print parser.print_help()
parser.error("No action entered")
action = args[0]
try:
if action == SETUP_ACTION:
setup(options)
elif action == START_ACTION:
start(options)
elif action == STOP_ACTION:
stop()
elif action == RESTART_ACTION:
stop()
start(options)
elif action == STATUS_ACTION:
svcstatus(options)
else:
parser.error("Invalid action")
if options.warnings:
for warning in options.warnings:
print_warning_msg(warning)
pass
options.exit_message = "Ambari Metrics Collector '%s' completed with warnings." % action
pass
except FatalException as e:
if e.reason is not None:
print_error_msg("Exiting with exit code {0}. \nREASON: {1}".format(e.code, e.reason))
sys.exit(e.code)
except NonFatalException as e:
options.exit_message = "Ambari Metrics Collector '%s' completed with warnings." % action
if e.reason is not None:
print_warning_msg(e.reason)
if options.exit_message is not None:
print options.exit_message
def update_ambari_properties():
prev_conf_file = search_file(configDefaults.AMBARI_PROPERTIES_BACKUP_FILE, get_conf_dir())
conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
# Previous config file does not exist
if (not prev_conf_file) or (prev_conf_file is None):
print_warning_msg("Can not find tbds.properties.backup file from previous version, skipping import of settings")
return 0
try:
old_properties = Properties()
old_properties.load(open(prev_conf_file))
except Exception, e:
print 'Could not read "%s": %s' % (prev_conf_file, e)
return -1
def _create_custom_user(self):
user = get_validated_string_input(
"Enter user account for ambari-server service ({0}):".format(self.user),
self.user, None,
"Invalid username.",
False
)
if user in self.NR_SYSTEM_USERS:
self.user = user
return 0
if get_silent():
password = self.password
else:
password = get_validated_string_input("Enter password for user {0}:".format(user), "", None, "Password", True, False)
from ambari_commons.os_windows import UserHelper
uh = UserHelper(user)
if uh.find_user():
print_info_msg("User {0} already exists, make sure that you typed correct password for user, "
"skipping user creation".format(user))
else:
status, message = uh.create_user(password)
if status == UserHelper.USER_EXISTS:
print_info_msg("User {0} already exists, make sure that you typed correct password for user, "
"skipping user creation".format(user))
elif status == UserHelper.ACTION_FAILED: # fail
print_warning_msg("Can't create user {0}. Failed with message {1}".format(user, message))
return UserHelper.ACTION_FAILED
self.password = password
# setting SeServiceLogonRight and SeBatchLogonRight to user
#This is unconditional
status, message = uh.add_user_privilege('SeServiceLogonRight')
if status == UserHelper.ACTION_FAILED:
print_warning_msg("Can't add SeServiceLogonRight to user {0}. Failed with message {1}".format(user, message))
return UserHelper.ACTION_FAILED
status, message = uh.add_user_privilege('SeBatchLogonRight')
if status == UserHelper.ACTION_FAILED:
print_warning_msg("Can't add SeBatchLogonRight to user {0}. Failed with message {1}".format(user, message))
return UserHelper.ACTION_FAILED
print_info_msg("User configuration is done.")
print_warning_msg("When using non SYSTEM user make sure that your user has read\write access to log directories and "
"all server directories. In case of integrated authentication for SQL Server make sure that your "
"user is properly configured to access the ambari database.")
if user.find('\\') == -1:
user = '******' + user
self.user = user
return 0
def generate_env(ambari_user, current_user):
properties = get_ambari_properties()
isSecure = get_is_secure(properties)
(isPersisted, masterKeyFile) = get_is_persisted(properties)
environ = os.environ.copy()
# Need to handle master key not persisted scenario
if isSecure and not masterKeyFile:
prompt = False
masterKey = environ.get(SECURITY_KEY_ENV_VAR_NAME)
if masterKey is not None and masterKey != "":
pass
else:
keyLocation = environ.get(SECURITY_MASTER_KEY_LOCATION)
if keyLocation is not None:
try:
# Verify master key can be read by the java process
with open(keyLocation, 'r'):
pass
except IOError:
print_warning_msg("Cannot read Master key from path specified in "
"environemnt.")
prompt = True
else:
# Key not provided in the environment
prompt = True
if prompt:
import pwd
masterKey = get_original_master_key(properties)
tempDir = tempfile.gettempdir()
tempFilePath = tempDir + os.sep + "masterkey"
save_master_key(masterKey, tempFilePath, True)
if ambari_user != current_user:
uid = pwd.getpwnam(ambari_user).pw_uid
gid = pwd.getpwnam(ambari_user).pw_gid
os.chown(tempFilePath, uid, gid)
else:
os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE)
if tempFilePath is not None:
environ[SECURITY_MASTER_KEY_LOCATION] = tempFilePath
return environ
def update_database_name_property(upgrade=False):
try:
check_database_name_property(upgrade)
except FatalException:
properties = get_ambari_properties()
if properties == -1:
err = "Error getting tbds properties"
raise FatalException(-1, err)
print_warning_msg(JDBC_DATABASE_NAME_PROPERTY + " property isn't set in " +
AMBARI_PROPERTIES_FILE + ". Setting it to default value - " + configDefaults.DEFAULT_DB_NAME)
properties.process_pair(JDBC_DATABASE_NAME_PROPERTY, configDefaults.DEFAULT_DB_NAME)
conf_file = find_properties_file()
try:
properties.store(open(conf_file, "w"))
except Exception, e:
err = 'Could not write tbds config file "%s": %s' % (conf_file, e)
raise FatalException(-1, err)
def os_is_root():
'''
Checks whether the current user is a member of the Administrators group
Returns True if yes, otherwise False
'''
retcode, out, err = os_run_os_command(WHOAMI_GROUPS)
if retcode != 0:
err_msg = "Unable to check the current user's group memberships. " \
"Command {0} returned exit code {1} with message: {2}".format(WHOAMI_GROUPS, retcode, err)
print_warning_msg(err_msg)
raise FatalException(retcode, err_msg)
#Check for Administrators group membership
if -1 != out.find('\n' + ADMIN_ACCOUNT):
return True
return False
def adjust_directory_permissions(ambari_user):
properties = get_ambari_properties()
bootstrap_dir = os.path.abspath(get_value_from_properties(properties, BOOTSTRAP_DIR_PROPERTY))
print_info_msg("Cleaning bootstrap directory ({0}) contents...".format(bootstrap_dir))
shutil.rmtree(bootstrap_dir, True) #Ignore the non-existent dir error
#Protect against directories lingering around
del_attempts = 0
while os.path.exists(bootstrap_dir) and del_attempts < 100:
time.sleep(50)
del_attempts += 1
if not os.path.exists(bootstrap_dir):
try:
os.makedirs(bootstrap_dir)
except Exception, ex:
print_warning_msg("Failed recreating the bootstrap directory: {0}".format(str(ex)))
pass
def check_result(self):
if self.returncode != 0:
print_warning_msg("Unable to check firewall status:{0}".format(self.stderrdata))
return False
profiles_status = [i for i in self.stdoutdata.split("\n") if not i == ""]
if "1" in profiles_status:
enabled_profiles = []
if profiles_status[0] == "1":
enabled_profiles.append("DomainProfile")
if profiles_status[1] == "1":
enabled_profiles.append("StandardProfile")
if profiles_status[2] == "1":
enabled_profiles.append("PublicProfile")
print_warning_msg(
"Following firewall profiles are enabled:{0}. Make sure that the firewall is properly configured.".format(
",".join(enabled_profiles)))
return True
return False
def update_ambari_env():
prev_env_file = search_file(configDefaults.AMBARI_ENV_BACKUP_FILE, configDefaults.DEFAULT_VLIBS_DIR)
env_file = search_file(AMBARI_ENV_FILE, configDefaults.DEFAULT_VLIBS_DIR)
# Previous env file does not exist
if (not prev_env_file) or (prev_env_file is None):
print_warning_msg("Can not find %s file from previous version, skipping restore of environment settings" %
configDefaults.AMBARI_ENV_BACKUP_FILE)
return 0
try:
if env_file is not None:
os.remove(env_file)
os.rename(prev_env_file, env_file)
print_warning_msg("Original file %s kept" % AMBARI_ENV_FILE)
except OSError as e:
print "Couldn't move %s file: %s" % (prev_env_file, e)
return -1
return 0
def update_ambari_properties():
prev_conf_file = search_file(configDefaults.AMBARI_PROPERTIES_BACKUP_FILE, get_conf_dir())
conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
# Previous config file does not exist
if (not prev_conf_file) or (prev_conf_file is None):
print_warning_msg("Can not find %s file from previous version, skipping import of settings" % configDefaults.AMBARI_PROPERTIES_BACKUP_FILE)
return 0
# ambari.properties file does not exists
if conf_file is None:
print_error_msg("Can't find %s file" % AMBARI_PROPERTIES_FILE)
return -1
with open(prev_conf_file) as hfOld:
try:
old_properties = Properties()
old_properties.load(hfOld)
except Exception, e:
print 'Could not read "%s": %s' % (prev_conf_file, e)
return -1
def _reset_local_database(self):
#force reset if silent option provided
if get_silent():
default = "yes"
else:
default = "no"
# Run automatic reset only for embedded DB
okToRun = get_YN_input("Confirm server reset [yes/no]({0})? ".format(default), get_silent())
if not okToRun:
err = "Ambari Server 'reset' cancelled"
raise FatalException(1, err)
print "Resetting the Server database..."
dbname = self.database_name
filename = self.drop_tables_script_file
username = self.database_username
password = self.database_password
command = PGConfig.SETUP_DB_CMD[:]
command[-1] = command[-1].format(filename, username, password, dbname)
drop_retcode, drop_outdata, drop_errdata = run_os_command(command)
if not drop_retcode == 0:
raise FatalException(1, drop_errdata)
if drop_errdata and PGConfig.PG_ERROR_BLOCKED in drop_errdata:
raise FatalException(1, "Database is in use. Please, make sure all connections to the database are closed")
if drop_errdata and get_verbose():
print_warning_msg(drop_errdata)
print_info_msg("About to run database setup")
retcode, outdata, errdata = self._setup_db()
if errdata and get_verbose():
print_warning_msg(errdata)
if (errdata and 'ERROR' in errdata.upper()) or (drop_errdata and 'ERROR' in drop_errdata.upper()):
err = "Non critical error in DDL"
if not get_verbose():
err += ", use --verbose for more information"
raise NonFatalException(err)
def is_valid_cert_host(certInfoDict):
if certInfoDict.has_key(COMMON_NAME_ATTR):
commonName = certInfoDict[COMMON_NAME_ATTR]
else:
print_warning_msg('There is no Common Name in Certificate')
return False
fqdn = get_fqdn()
if not fqdn:
print_warning_msg('Failed to get server FQDN')
return False
if commonName != fqdn:
print_warning_msg('Common Name in Certificate: ' + commonName + ' does not match the server FQDN: ' + fqdn)
return False
return True
def upgrade(args):
print_info_msg("Upgrade Ambari Server", True)
if not is_root():
err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT
raise FatalException(4, err)
print_info_msg('Updating Ambari Server properties in {0} ...'.format(AMBARI_PROPERTIES_FILE), True)
retcode = update_ambari_properties()
if not retcode == 0:
err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
print_info_msg('Updating Ambari Server properties in {0} ...'.format(AMBARI_ENV_FILE), True)
retcode = update_ambari_env()
if not retcode == 0:
err = AMBARI_ENV_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
retcode = update_krb_jaas_login_properties()
if retcode == -2:
pass # no changes done, let's be silent
elif retcode == 0:
print_info_msg("File {0} updated.".format(AMBARI_KRB_JAAS_LOGIN_FILE), True)
elif not retcode == 0:
err = AMBARI_KRB_JAAS_LOGIN_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
restore_custom_services()
replay_mpack_logs()
try:
update_database_name_property(upgrade=True)
except FatalException:
return -1
# Ignore the server version & database options passed via command-line arguments
parse_properties_file(args)
#TODO check database version
change_objects_owner(args)
retcode = run_schema_upgrade(args)
if not retcode == 0:
print_error_msg("Ambari server upgrade failed. Please look at {0}, for more details.".format(configDefaults.SERVER_LOG_FILE))
raise FatalException(11, 'Schema upgrade failed.')
user = read_ambari_user()
if user is None:
warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG
print_warning_msg(warn)
else:
adjust_directory_permissions(user)
# local repo
upgrade_local_repo(args)
# create jdbc symlinks if jdbc drivers are available in resources
check_jdbc_drivers(args)
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
# Move *.py files from custom_actions to custom_actions/scripts
# This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0
ambari_version = get_ambari_version(properties)
if ambari_version is None:
args.warnings.append("*.py files were not moved from custom_actions to custom_actions/scripts.")
elif compare_versions(ambari_version, "2.0.0") == 0:
move_user_custom_actions()
# Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0
admin_views_dirs = get_admin_views_dir(properties)
for admin_views_dir in admin_views_dirs:
shutil.rmtree(admin_views_dir)
# Modify timestamp of views jars to current time
views_jars = get_views_jars(properties)
for views_jar in views_jars:
os.utime(views_jar, None)
# check if ambari has obsolete LDAP configuration
if properties.get_property(LDAP_PRIMARY_URL_PROPERTY) and not properties.get_property(IS_LDAP_CONFIGURED):
args.warnings.append("Existing LDAP configuration is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration.")
# adding custom jdbc name and previous custom jdbc properties
# we need that to support new dynamic jdbc names for upgraded ambari
add_jdbc_properties(properties)
def generate_child_process_param_list(ambari_user, current_user, java_exe,
class_path, debug_start, suspend_mode):
from ambari_commons.os_linux import ULIMIT_CMD
properties = get_ambari_properties()
isSecure = get_is_secure(properties)
(isPersisted, masterKeyFile) = get_is_persisted(properties)
environ = os.environ.copy()
# Need to handle master key not persisted scenario
if isSecure and not masterKeyFile:
prompt = False
masterKey = environ.get(SECURITY_KEY_ENV_VAR_NAME)
if masterKey is not None and masterKey != "":
pass
else:
keyLocation = environ.get(SECURITY_MASTER_KEY_LOCATION)
if keyLocation is not None:
try:
# Verify master key can be read by the java process
with open(keyLocation, 'r'):
pass
except IOError:
print_warning_msg(
"Cannot read Master key from path specified in "
"environemnt.")
prompt = True
else:
# Key not provided in the environment
prompt = True
if prompt:
import pwd
masterKey = get_original_master_key(properties)
tempDir = tempfile.gettempdir()
tempFilePath = tempDir + os.sep + "masterkey"
save_master_key(masterKey, tempFilePath, True)
if ambari_user != current_user:
uid = pwd.getpwnam(ambari_user).pw_uid
gid = pwd.getpwnam(ambari_user).pw_gid
os.chown(tempFilePath, uid, gid)
else:
os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE)
if tempFilePath is not None:
environ[SECURITY_MASTER_KEY_LOCATION] = tempFilePath
command_base = SERVER_START_CMD_DEBUG if debug_start else SERVER_START_CMD
ulimit_cmd = "%s %s" % (ULIMIT_CMD, str(get_ulimit_open_files(properties)))
command = command_base.format(
java_exe, ambari_provider_module_option, jvm_args, class_path,
configDefaults.SERVER_OUT_FILE,
os.path.join(configDefaults.PID_DIR, EXITCODE_NAME), suspend_mode)
# required to start properly server instance
os.chdir(configDefaults.ROOT_FS_PATH)
#For properly daemonization server should be started using shell as parent
param_list = [locate_file('sh', '/bin'), "-c"]
if is_root() and ambari_user != "root":
# To inherit exported environment variables (especially AMBARI_PASSPHRASE),
# from subprocess, we have to skip --login option of su command. That's why
# we change dir to / (otherwise subprocess can face with 'permission denied'
# errors while trying to list current directory
cmd = "{ulimit_cmd} ; {su} {ambari_user} -s {sh_shell} -c '{command}'".format(
ulimit_cmd=ulimit_cmd,
su=locate_file('su', '/bin'),
ambari_user=ambari_user,
sh_shell=locate_file('sh', '/bin'),
command=command)
else:
cmd = "{ulimit_cmd} ; {command}".format(ulimit_cmd=ulimit_cmd,
command=command)
param_list.append(cmd)
return (param_list, environ)
def server_process_main(options, scmStatus=None):
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
raise FatalException(-1, err)
properties_for_print = []
logger.info("Ambari server properties config:")
for key, value in properties.getPropertyDict().items():
if "passwd" not in key and "password" not in key:
properties_for_print.append(key + "=" + value)
logger.info(properties_for_print)
# debug mode, including stop Java process at startup
try:
set_debug_mode_from_options(options)
except AttributeError:
pass
if not check_reverse_lookup():
print_warning_msg(
"The hostname was not found in the reverse DNS lookup. "
"This may result in incorrect behavior. "
"Please check the DNS setup and fix the issue.")
check_database_name_property()
parse_properties_file(options)
is_active_instance = get_is_active_instance()
if not is_active_instance:
print_warning_msg(
"This instance of ambari server is not designated as active. Cannot start ambari server."
)
err = "This is not an active instance. Shutting down..."
raise FatalException(1, err)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
print_info_msg("Ambari Server is not running...")
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
if not options.skip_properties_validation:
missing_properties = get_missing_properties(properties)
if missing_properties:
err = "Required properties are not found: " + str(missing_properties) + ". To skip properties validation " \
"use \"--skip-properties-validation\""
raise FatalException(1, err)
# Preparations
if is_root():
print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT
ensure_jdbc_driver_is_installed(options, properties)
ensure_dbms_is_running(options, properties, scmStatus)
if scmStatus is not None:
scmStatus.reportStartPending()
refresh_stack_hash(properties)
if scmStatus is not None:
scmStatus.reportStartPending()
ensure_server_security_is_configured()
if scmStatus is not None:
scmStatus.reportStartPending()
java_exe = get_java_exe_path()
serverClassPath = ServerClassPath(properties, options)
debug_mode = get_debug_mode()
debug_start = (debug_mode & 1) or SERVER_START_DEBUG
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
environ = generate_env(options, ambari_user, current_user)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell(
validate_classpath=True)
if options.skip_database_check:
global jvm_args
jvm_args += " -DskipDatabaseConsistencyCheck"
print "Ambari Server is starting with the database consistency check skipped. Do not make any changes to your cluster " \
"topology or perform a cluster upgrade until you correct the database consistency issues. See \"" \
+ configDefaults.DB_CHECK_LOG + "\" for more details on the consistency issues."
properties.process_pair(CHECK_DATABASE_SKIPPED_PROPERTY, "true")
else:
print "Ambari database consistency check started..."
properties.process_pair(CHECK_DATABASE_SKIPPED_PROPERTY, "false")
command = CHECK_DATABASE_HELPER_CMD.format(java_exe, class_path)
(retcode, stdout, stderr) = run_os_command(command, env=environ)
if retcode > 0:
print str(stdout)
raise FatalException(
1, 'Database check failed to complete. Please check ' +
configDefaults.SERVER_LOG_FILE + ' and ' +
configDefaults.DB_CHECK_LOG + ' for more information.')
else:
print str(stdout)
print "Ambari database consistency check finished"
if not stdout.startswith("No errors"):
sys.exit(1)
update_properties(properties)
param_list = generate_child_process_param_list(ambari_user, java_exe,
class_path, debug_start,
suspend_mode)
# The launched shell process and sub-processes should have a group id that
# is different from the parent.
def make_process_independent():
if IS_FOREGROUND: # upstart script is not able to track process from different pgid.
return
processId = os.getpid()
if processId > 0:
try:
os.setpgid(processId, processId)
except OSError, e:
print_warning_msg('setpgid({0}, {0}) failed - {1}'.format(
pidJava, str(e)))
pass
def download_and_install_jdk(self, args, properties):
conf_file = properties.fileName
jcePolicyWarn = "JCE Policy files are required for configuring Kerberos security. If you plan to use Kerberos," \
"please make sure JCE Unlimited Strength Jurisdiction Policy Files are valid on all hosts."
if args.java_home:
#java_home was specified among the command-line arguments. Use it as custom JDK location.
if not validate_jdk(args.java_home):
err = "Path to java home " + args.java_home + " or java binary file does not exists"
raise FatalException(1, err)
print_warning_msg("JAVA_HOME " + args.java_home + " must be valid on ALL hosts")
print_warning_msg(jcePolicyWarn)
IS_CUSTOM_JDK = True
properties.process_pair(JAVA_HOME_PROPERTY, args.java_home)
properties.removeOldProp(JDK_NAME_PROPERTY)
properties.removeOldProp(JCE_NAME_PROPERTY)
self._ensure_java_home_env_var_is_set(args.java_home)
self.jdk_index = self.custom_jdk_number
return
java_home_var = get_JAVA_HOME()
if get_silent():
if not java_home_var:
#No java_home_var set, detect if java is already installed
if os.environ.has_key(JAVA_HOME):
args.java_home = os.environ[JAVA_HOME]
properties.process_pair(JAVA_HOME_PROPERTY, args.java_home)
properties.removeOldProp(JDK_NAME_PROPERTY)
properties.removeOldProp(JCE_NAME_PROPERTY)
self._ensure_java_home_env_var_is_set(args.java_home)
self.jdk_index = self.custom_jdk_number
return
else:
# For now, changing the existing JDK to make sure we use a supported one
pass
if java_home_var:
change_jdk = get_YN_input("Do you want to change Oracle JDK [y/n] (n)? ", False)
if not change_jdk:
self._ensure_java_home_env_var_is_set(java_home_var)
self.jdk_index = self.custom_jdk_number
return
#Continue with the normal setup, taking the first listed JDK version as the default option
jdk_num = str(self.jdk_index + 1)
(self.jdks, jdk_choice_prompt, jdk_valid_choices, self.custom_jdk_number) = self._populate_jdk_configs(properties, jdk_num)
jdk_num = get_validated_string_input(
jdk_choice_prompt,
jdk_num,
jdk_valid_choices,
"Invalid number.",
False
)
self.jdk_index = int(jdk_num) - 1
if self.jdk_index == self.custom_jdk_number:
print_warning_msg("JDK must be installed on all hosts and JAVA_HOME must be valid on all hosts.")
print_warning_msg(jcePolicyWarn)
args.java_home = get_validated_string_input("Path to JAVA_HOME: ", None, None, None, False, False)
if not os.path.exists(args.java_home) or not os.path.isfile(os.path.join(args.java_home, "bin", self.JAVA_BIN)):
err = "Java home path or java binary file is unavailable. Please put correct path to java home."
raise FatalException(1, err)
print "Validating JDK on Ambari Server...done."
properties.process_pair(JAVA_HOME_PROPERTY, args.java_home)
properties.removeOldProp(JDK_NAME_PROPERTY)
properties.removeOldProp(JCE_NAME_PROPERTY)
self._ensure_java_home_env_var_is_set(args.java_home)
return
jdk_cfg = self.jdks[self.jdk_index]
try:
resources_dir = properties[RESOURCES_DIR_PROPERTY]
except (KeyError), e:
err = 'Property ' + str(e) + ' is not defined at ' + conf_file
raise FatalException(1, err)
def import_cert_and_key(security_server_keys_dir):
import_cert_path = get_validated_filepath_input( \
"Enter path to Certificate: ", \
"Certificate not found")
import_key_path = get_validated_filepath_input( \
"Enter path to Private Key: ", "Private Key not found")
pem_password = get_validated_string_input("Please enter password for Private Key: ", "", None, None, True)
certInfoDict = get_cert_info(import_cert_path)
if not certInfoDict:
print_warning_msg('Unable to get Certificate information')
else:
#Validate common name of certificate
if not is_valid_cert_host(certInfoDict):
print_warning_msg('Unable to validate Certificate hostname')
#Validate issue and expirations dates of certificate
if not is_valid_cert_exp(certInfoDict):
print_warning_msg('Unable to validate Certificate issue and expiration dates')
#jetty requires private key files with non-empty key passwords
retcode = 0
err = ''
if not pem_password:
print 'Generating random password for HTTPS keystore...done.'
pem_password = generate_random_string()
retcode, out, err = run_os_command(CHANGE_KEY_PWD_CND.format(
import_key_path, pem_password))
import_key_path += '.secured'
if retcode == 0:
keystoreFilePath = os.path.join(security_server_keys_dir, \
SSL_KEYSTORE_FILE_NAME)
keystoreFilePathTmp = os.path.join(tempfile.gettempdir(), \
SSL_KEYSTORE_FILE_NAME)
passFilePath = os.path.join(security_server_keys_dir, \
SSL_KEY_PASSWORD_FILE_NAME)
passFilePathTmp = os.path.join(tempfile.gettempdir(), \
SSL_KEY_PASSWORD_FILE_NAME)
passinFilePath = os.path.join(tempfile.gettempdir(), \
SSL_PASSIN_FILE)
passwordFilePath = os.path.join(tempfile.gettempdir(), \
SSL_PASSWORD_FILE)
with open(passFilePathTmp, 'w+') as passFile:
passFile.write(pem_password)
passFile.close
pass
set_file_permissions(passFilePath, "660", read_ambari_user(), False)
copy_file(passFilePathTmp, passinFilePath)
copy_file(passFilePathTmp, passwordFilePath)
retcode, out, err = run_os_command(EXPRT_KSTR_CMD.format(import_cert_path, \
import_key_path, passwordFilePath, passinFilePath, keystoreFilePathTmp))
if retcode == 0:
print 'Importing and saving Certificate...done.'
import_file_to_keystore(keystoreFilePathTmp, keystoreFilePath)
import_file_to_keystore(passFilePathTmp, passFilePath)
import_file_to_keystore(import_cert_path, os.path.join( \
security_server_keys_dir, SSL_CERT_FILE_NAME))
import_file_to_keystore(import_key_path, os.path.join( \
security_server_keys_dir, SSL_KEY_FILE_NAME))
#Validate keystore
retcode, out, err = run_os_command(VALIDATE_KEYSTORE_CMD.format(keystoreFilePath, \
passwordFilePath, passinFilePath))
remove_file(passinFilePath)
remove_file(passwordFilePath)
if not retcode == 0:
print 'Error during keystore validation occured!:'
print err
return False
return True
else:
print_error_msg('Could not import Certificate and Private Key.')
print 'SSL error on exporting keystore: ' + err.rstrip() + \
'.\nPlease ensure that provided Private Key password is correct and ' + \
're-import Certificate.'
return False
def server_process_main(options, scmStatus=None):
# debug mode, including stop Java process at startup
try:
set_debug_mode_from_options(options)
except AttributeError:
pass
if not check_reverse_lookup():
print_warning_msg("The hostname was not found in the reverse DNS lookup. "
"This may result in incorrect behavior. "
"Please check the DNS setup and fix the issue.")
check_database_name_property()
parse_properties_file(options)
is_active_instance = get_is_active_instance()
if not is_active_instance:
print_warning_msg("This instance of ambari server is not designated as active. Cannot start ambari server.")
err = "This is not an active instance. Shutting down..."
raise FatalException(1, err)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
print_info_msg("Ambari Server is not running...")
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
properties = get_ambari_properties()
# Preparations
if is_root():
print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT
ensure_jdbc_driver_is_installed(options, properties)
ensure_dbms_is_running(options, properties, scmStatus)
if scmStatus is not None:
scmStatus.reportStartPending()
refresh_stack_hash(properties)
if scmStatus is not None:
scmStatus.reportStartPending()
ensure_server_security_is_configured()
if scmStatus is not None:
scmStatus.reportStartPending()
java_exe = get_java_exe_path()
class_path = get_conf_dir()
class_path = os.path.abspath(class_path) + os.pathsep + get_ambari_classpath()
jdbc_driver_path = get_jdbc_driver_path(options, properties)
if jdbc_driver_path not in class_path:
class_path = class_path + os.pathsep + jdbc_driver_path
if SERVER_CLASSPATH_KEY in os.environ:
class_path = os.environ[SERVER_CLASSPATH_KEY] + os.pathsep + class_path
native_libs_path = get_native_libs_path(options, properties)
if native_libs_path is not None:
if LIBRARY_PATH_KEY in os.environ:
native_libs_path = os.environ[LIBRARY_PATH_KEY] + os.pathsep + native_libs_path
os.environ[LIBRARY_PATH_KEY] = native_libs_path
debug_mode = get_debug_mode()
debug_start = (debug_mode & 1) or SERVER_START_DEBUG
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
param_list = generate_child_process_param_list(ambari_user, java_exe,
class_path, debug_start,
suspend_mode)
environ = generate_env(ambari_user, current_user)
if not os.path.exists(configDefaults.PID_DIR):
os.makedirs(configDefaults.PID_DIR, 0755)
print_info_msg("Running server: " + str(param_list))
procJava = subprocess.Popen(param_list, env=environ)
pidJava = procJava.pid
if pidJava <= 0:
procJava.terminate()
exitcode = procJava.returncode
exitfile = os.path.join(configDefaults.PID_DIR, EXITCODE_NAME)
save_pid(exitcode, exitfile)
if scmStatus is not None:
scmStatus.reportStopPending()
raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, configDefaults.SERVER_OUT_FILE))
else:
# Change the group id to the process id of the parent so that the launched
# process and sub-processes have a group id that is different from the parent.
try:
os.setpgid(pidJava, 0)
except OSError, e:
print_warning_msg('setpgid({0}, 0) failed - {1}'.format(pidJava, str(e)))
pass
pidfile = os.path.join(configDefaults.PID_DIR, PID_NAME)
save_pid(pidJava, pidfile)
print "Server PID at: "+pidfile
print "Server out at: "+configDefaults.SERVER_OUT_FILE
print "Server log at: "+configDefaults.SERVER_LOG_FILE
wait_for_server_start(pidfile, scmStatus)
def upgrade(args):
print_info_msg("Upgrade Ambari Server", True)
if not is_root():
err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT
raise FatalException(4, err)
print_info_msg(
'Updating Ambari Server properties in {0} ...'.format(
AMBARI_PROPERTIES_FILE), True)
retcode = update_ambari_properties()
if not retcode == 0:
err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
print_info_msg(
'Updating Ambari Server properties in {0} ...'.format(AMBARI_ENV_FILE),
True)
retcode = update_ambari_env()
if not retcode == 0:
err = AMBARI_ENV_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
retcode = update_krb_jaas_login_properties()
if retcode == -2:
pass # no changes done, let's be silent
elif retcode == 0:
print_info_msg("File {0} updated.".format(AMBARI_KRB_JAAS_LOGIN_FILE),
True)
elif not retcode == 0:
err = AMBARI_KRB_JAAS_LOGIN_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
restore_custom_services()
replay_mpack_logs()
try:
update_database_name_property(upgrade=True)
except FatalException:
return -1
# Ignore the server version & database options passed via command-line arguments
parse_properties_file(args)
#TODO check database version
change_objects_owner(args)
retcode = run_schema_upgrade(args)
if not retcode == 0:
print_error_msg(
"Ambari server upgrade failed. Please look at {0}, for more details."
.format(configDefaults.SERVER_LOG_FILE))
raise FatalException(11, 'Schema upgrade failed.')
user = read_ambari_user()
if user is None:
warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG
print_warning_msg(warn)
else:
adjust_directory_permissions(user)
# create jdbc symlinks if jdbc drivers are available in resources
check_jdbc_drivers(args)
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
# Move *.py files from custom_actions to custom_actions/scripts
# This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0
ambari_version = get_ambari_version(properties)
if ambari_version is None:
args.warnings.append(
"*.py files were not moved from custom_actions to custom_actions/scripts."
)
elif compare_versions(ambari_version, "2.0.0") == 0:
move_user_custom_actions()
# Move files installed by package to default views directory to a custom one
for views_dir in get_views_dir(properties):
root_views_dir = views_dir + "/../"
if os.path.samefile(root_views_dir, get_default_views_dir()):
continue
for file in glob.glob(get_default_views_dir() + '/*'):
shutil.move(file, root_views_dir)
# Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0
admin_views_dirs = get_admin_views_dir(properties)
for admin_views_dir in admin_views_dirs:
shutil.rmtree(admin_views_dir)
# Modify timestamp of views jars to current time
views_jars = get_views_jars(properties)
for views_jar in views_jars:
os.utime(views_jar, None)
# check if ambari is configured to use LDAP authentication
if properties.get_property(CLIENT_SECURITY) == "ldap":
args.warnings.append(
"LDAP authentication is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration."
)
# adding custom jdbc name and previous custom jdbc properties
# we need that to support new dynamic jdbc names for upgraded ambari
add_jdbc_properties(properties)
json_url = get_json_url_from_repo_file()
if json_url:
print "Ambari repo file contains latest json url {0}, updating stacks repoinfos with it...".format(
json_url)
properties = get_ambari_properties()
stack_root = get_stack_location(properties)
update_latest_in_repoinfos_for_all_stacks(stack_root, json_url)
else:
print "Ambari repo file doesn't contain latest json url, skipping repoinfos modification"
def server_process_main(options, scmStatus=None):
# debug mode, including stop Java process at startup
try:
set_debug_mode_from_options(options)
except AttributeError:
pass
if not check_reverse_lookup():
print_warning_msg("The hostname was not found in the reverse DNS lookup. "
"This may result in incorrect behavior. "
"Please check the DNS setup and fix the issue.")
check_database_name_property()
parse_properties_file(options)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
print_info_msg("Ambari Server is not running...")
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
properties = get_ambari_properties()
# Preparations
if is_root():
print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT
ensure_jdbc_driver_is_installed(options, properties)
ensure_dbms_is_running(options, properties, scmStatus)
if scmStatus is not None:
scmStatus.reportStartPending()
refresh_stack_hash(properties)
if scmStatus is not None:
scmStatus.reportStartPending()
ensure_server_security_is_configured()
if scmStatus is not None:
scmStatus.reportStartPending()
java_exe = get_java_exe_path()
serverClassPath = ServerClassPath(properties, options)
debug_mode = get_debug_mode()
debug_start = (debug_mode & 1) or SERVER_START_DEBUG
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
param_list = generate_child_process_param_list(ambari_user, java_exe,
serverClassPath.get_full_ambari_classpath_escaped_for_shell(), debug_start,
suspend_mode)
environ = generate_env(options, ambari_user, current_user)
if not os.path.exists(configDefaults.PID_DIR):
os.makedirs(configDefaults.PID_DIR, 0755)
# The launched shell process and sub-processes should have a group id that
# is different from the parent.
def make_process_independent():
if IS_FOREGROUND: # upstart script is not able to track process from different pgid.
return
processId = os.getpid()
if processId > 0:
try:
os.setpgid(processId, processId)
except OSError, e:
print_warning_msg('setpgid({0}, {0}) failed - {1}'.format(pidJava, str(e)))
pass
def server_process_main(options, scmStatus=None):
# debug mode, including stop Java process at startup
try:
set_debug_mode_from_options(options)
except AttributeError:
pass
if not check_reverse_lookup():
print_warning_msg(
"The hostname was not found in the reverse DNS lookup. "
"This may result in incorrect behavior. "
"Please check the DNS setup and fix the issue.")
check_database_name_property()
parse_properties_file(options)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
print_info_msg("Ambari Server is not running...")
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
properties = get_ambari_properties()
# Preparations
if is_root():
print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT
ensure_jdbc_driver_is_installed(options, properties)
ensure_dbms_is_running(options, properties, scmStatus)
if scmStatus is not None:
scmStatus.reportStartPending()
refresh_stack_hash(properties)
if scmStatus is not None:
scmStatus.reportStartPending()
ensure_server_security_is_configured()
if scmStatus is not None:
scmStatus.reportStartPending()
java_exe = get_java_exe_path()
class_path = get_conf_dir()
class_path = os.path.abspath(
class_path) + os.pathsep + get_ambari_classpath()
debug_mode = get_debug_mode()
debug_start = (debug_mode & 1) or SERVER_START_DEBUG
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
(param_list,
environ) = generate_child_process_param_list(ambari_user, current_user,
java_exe, class_path,
debug_start, suspend_mode)
if not os.path.exists(configDefaults.PID_DIR):
os.makedirs(configDefaults.PID_DIR, 0755)
print_info_msg("Running server: " + str(param_list))
procJava = subprocess.Popen(param_list, env=environ)
pidJava = procJava.pid
if pidJava <= 0:
procJava.terminate()
exitcode = procJava.returncode
exitfile = os.path.join(configDefaults.PID_DIR, EXITCODE_NAME)
save_pid(exitcode, exitfile)
if scmStatus is not None:
scmStatus.reportStopPending()
raise FatalException(
-1,
AMBARI_SERVER_DIE_MSG.format(exitcode,
configDefaults.SERVER_OUT_FILE))
else:
pidfile = os.path.join(configDefaults.PID_DIR, PID_NAME)
save_pid(pidJava, pidfile)
print "Server PID at: " + pidfile
print "Server out at: " + configDefaults.SERVER_OUT_FILE
print "Server log at: " + configDefaults.SERVER_LOG_FILE
wait_for_server_start(pidfile, scmStatus)
if scmStatus is not None:
scmStatus.reportStarted()
return procJava
def upgrade(args):
if not is_root():
err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT
raise FatalException(4, err)
print 'Updating properties in ' + AMBARI_PROPERTIES_FILE + ' ...'
retcode = update_ambari_properties()
if not retcode == 0:
err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
try:
update_database_name_property(upgrade=True)
except FatalException:
return -1
# Ignore the server version & database options passed via command-line arguments
parse_properties_file(args)
#TODO check database version
change_objects_owner(args)
retcode = run_schema_upgrade()
if not retcode == 0:
print_error_msg("Ambari server upgrade failed. Please look at {0}, for more details.".format(configDefaults.SERVER_LOG_FILE))
raise FatalException(11, 'Schema upgrade failed.')
user = read_ambari_user()
if user is None:
warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG
print_warning_msg(warn)
else:
adjust_directory_permissions(user)
# local repo
upgrade_local_repo(args)
# create jdbc symlinks if jdbc drivers are available in resources
check_jdbc_drivers(args)
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
# Move *.py files from custom_actions to custom_actions/scripts
# This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0
ambari_version = get_ambari_version(properties)
if ambari_version is None:
args.warnings.append("*.py files were not moved from custom_actions to custom_actions/scripts.")
elif compare_versions(ambari_version, "2.0.0") == 0:
move_user_custom_actions()
# Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0
admin_views_dirs = get_admin_views_dir(properties)
for admin_views_dir in admin_views_dirs:
shutil.rmtree(admin_views_dir)
# check if ambari has obsolete LDAP configuration
if properties.get_property(LDAP_PRIMARY_URL_PROPERTY) and not properties.get_property(IS_LDAP_CONFIGURED):
args.warnings.append("Existing LDAP configuration is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration.")
def main(options, args, parser):
init_logging()
# set silent
set_silent(options.silent)
# debug mode
set_debug_mode_from_options(options)
init_debug(options)
#perform checks
options.warnings = []
if are_cmd_line_db_args_blank(options):
options.must_set_database_options = True
elif not are_cmd_line_db_args_valid(options):
parser.error('All database options should be set. Please see help for the options.')
else:
options.must_set_database_options = False
#correct database
fix_database_options(options, parser)
if len(args) == 0:
print parser.print_help()
parser.error("No action entered")
action_map = create_user_action_map(args, options)
action = args[0]
try:
action_obj = action_map[action]
except KeyError:
parser.error("Invalid action: " + action)
matches = 0
for args_number_required in action_obj.possible_args_numbers:
matches += int(len(args) == args_number_required)
if matches == 0:
print parser.print_help()
possible_args = ' or '.join(str(x) for x in action_obj.possible_args_numbers)
parser.error("Invalid number of arguments. Entered: " + str(len(args)) + ", required: " + possible_args)
options.exit_message = "Ambari Server '%s' completed successfully." % action
options.exit_code = None
try:
action_obj.execute()
if action_obj.need_restart:
pstatus, pid = is_server_runing()
if pstatus:
print 'NOTE: Restart Ambari Server to apply changes' + \
' ("ambari-server restart|stop+start")'
if options.warnings:
for warning in options.warnings:
print_warning_msg(warning)
pass
options.exit_message = "Ambari Server '%s' completed with warnings." % action
pass
except FatalException as e:
if e.reason is not None:
print_error_msg("Exiting with exit code {0}. \nREASON: {1}".format(e.code, e.reason))
logger.exception(str(e))
sys.exit(e.code)
except NonFatalException as e:
options.exit_message = "Ambari Server '%s' completed with warnings." % action
if e.reason is not None:
print_warning_msg(e.reason)
if options.exit_message is not None:
print options.exit_message
if options.exit_code is not None: # not all actions may return a system exit code
sys.exit(options.exit_code)
#Protect against directories lingering around
del_attempts = 0
while os.path.exists(bootstrap_dir) and del_attempts < 100:
time.sleep(50)
del_attempts += 1
if not os.path.exists(bootstrap_dir):
try:
os.makedirs(bootstrap_dir)
except Exception, ex:
print_warning_msg(
"Failed recreating the bootstrap directory: {0}".format(
str(ex)))
pass
else:
print_warning_msg(
"Bootstrap directory lingering around after 5s. Unable to complete the cleanup."
)
pass
# Add master key and credential store if exists
keyLocation = get_master_key_location(properties)
masterKeyFile = search_file(SECURITY_MASTER_KEY_FILENAME, keyLocation)
if masterKeyFile:
configDefaults.NR_ADJUST_OWNERSHIP_LIST.append(
(masterKeyFile, configDefaults.MASTER_KEY_FILE_PERMISSIONS, "{0}",
False))
credStoreFile = get_credential_store_location(properties)
if os.path.exists(credStoreFile):
configDefaults.NR_ADJUST_OWNERSHIP_LIST.append(
(credStoreFile, configDefaults.CREDENTIALS_STORE_FILE_PERMISSIONS,
"{0}", False))
def setup_master_key(options):
if not is_root():
err = 'Ambari-server setup should be run with ' \
'root-level privileges'
raise FatalException(4, err)
properties = get_ambari_properties()
if properties == -1:
raise FatalException(1, "Failed to read properties file.")
db_windows_auth_prop = properties.get_property(
JDBC_USE_INTEGRATED_AUTH_PROPERTY)
db_sql_auth = False if db_windows_auth_prop and db_windows_auth_prop.lower(
) == 'true' else True
db_password = properties.get_property(JDBC_PASSWORD_PROPERTY)
# Encrypt passwords cannot be called before setup
if db_sql_auth and not db_password:
print 'Please call "setup" before "encrypt-passwords". Exiting...'
return 1
# Check configuration for location of master key
isSecure = get_is_secure(properties)
(isPersisted, masterKeyFile) = get_is_persisted(properties)
# Read clear text DB password from file
if db_sql_auth and not is_alias_string(db_password) and os.path.isfile(
db_password):
with open(db_password, 'r') as passwdfile:
db_password = passwdfile.read()
ldap_password = properties.get_property(LDAP_MGR_PASSWORD_PROPERTY)
if ldap_password:
# Read clear text LDAP password from file
if not is_alias_string(ldap_password) and os.path.isfile(
ldap_password):
with open(ldap_password, 'r') as passwdfile:
ldap_password = passwdfile.read()
ts_password = properties.get_property(SSL_TRUSTSTORE_PASSWORD_PROPERTY)
resetKey = False
masterKey = None
if isSecure:
print "Password encryption is enabled."
resetKey = get_YN_input("Do you want to reset Master Key? [y/n] (n): ",
False)
# For encrypting of only unencrypted passwords without resetting the key ask
# for master key if not persisted.
if isSecure and not isPersisted and not resetKey:
print "Master Key not persisted."
masterKey = get_original_master_key(properties)
pass
# Make sure both passwords are clear-text if master key is lost
if resetKey:
if not isPersisted:
print "Master Key not persisted."
masterKey = get_original_master_key(properties)
# Unable get the right master key or skipped question <enter>
if not masterKey:
print "To disable encryption, do the following:"
print "- Edit " + find_properties_file() + \
" and set " + SECURITY_IS_ENCRYPTION_ENABLED + " = " + "false."
err = "{0} is already encrypted. Please call {1} to store unencrypted" \
" password and call 'encrypt-passwords' again."
if db_sql_auth and db_password and is_alias_string(
db_password):
print err.format('- Database password',
"'" + SETUP_ACTION + "'")
if ldap_password and is_alias_string(ldap_password):
print err.format('- LDAP manager password',
"'" + LDAP_SETUP_ACTION + "'")
if ts_password and is_alias_string(ts_password):
print err.format('TrustStore password',
"'" + LDAP_SETUP_ACTION + "'")
return 1
pass
pass
pass
# Read back any encrypted passwords
if db_sql_auth and db_password and is_alias_string(db_password):
db_password = read_passwd_for_alias(JDBC_RCA_PASSWORD_ALIAS, masterKey)
if ldap_password and is_alias_string(ldap_password):
ldap_password = read_passwd_for_alias(LDAP_MGR_PASSWORD_ALIAS,
masterKey)
if ts_password and is_alias_string(ts_password):
ts_password = read_passwd_for_alias(SSL_TRUSTSTORE_PASSWORD_ALIAS,
masterKey)
# Read master key, if non-secure or reset is true
if resetKey or not isSecure:
masterKey = read_master_key(resetKey)
persist = get_YN_input("Do you want to persist master key. If you choose " \
"not to persist, you need to provide the Master " \
"Key while starting the ambari server as an env " \
"variable named " + SECURITY_KEY_ENV_VAR_NAME + \
" or the start will prompt for the master key."
" Persist [y/n] (y)? ", True)
if persist:
save_master_key(
options, masterKey,
get_master_key_location(properties) + os.sep +
SECURITY_MASTER_KEY_FILENAME, persist)
elif not persist and masterKeyFile:
try:
os.remove(masterKeyFile)
print_info_msg("Deleting master key file at location: " +
str(masterKeyFile))
except Exception, e:
print 'ERROR: Could not remove master key file. %s' % e
# Blow up the credential store made with previous key, if any
store_file = get_credential_store_location(properties)
if os.path.exists(store_file):
try:
os.remove(store_file)
except:
print_warning_msg("Failed to remove credential store file.")
pass
pass
def download_and_install_jdk(self, args, properties):
conf_file = properties.fileName
jcePolicyWarn = "JCE Policy files are required for configuring Kerberos security. If you plan to use Kerberos," \
"please make sure JCE Unlimited Strength Jurisdiction Policy Files are valid on all hosts."
if args.java_home:
#java_home was specified among the command-line arguments. Use it as custom JDK location.
if not validate_jdk(args.java_home):
err = "Path to java home " + args.java_home + " or java binary file does not exists"
raise FatalException(1, err)
print_warning_msg("JAVA_HOME " + args.java_home +
" must be valid on ALL hosts")
print_warning_msg(jcePolicyWarn)
IS_CUSTOM_JDK = True
properties.process_pair(JAVA_HOME_PROPERTY, args.java_home)
properties.removeOldProp(JDK_NAME_PROPERTY)
properties.removeOldProp(JCE_NAME_PROPERTY)
self._ensure_java_home_env_var_is_set(args.java_home)
self.jdk_index = self.custom_jdk_number
return
java_home_var = get_JAVA_HOME()
if OS_FAMILY == OSConst.WINSRV_FAMILY:
progress_func = None
else:
progress_func = download_progress
if java_home_var:
change_jdk = get_YN_input(
"Do you want to change Oracle JDK [y/n] (n)? ", False)
if not change_jdk:
self._ensure_java_home_env_var_is_set(java_home_var)
self.jdk_index = self.custom_jdk_number
return
#Continue with the normal setup, taking the first listed JDK version as the default option
jdk_num = str(self.jdk_index + 1)
(self.jdks, jdk_choice_prompt, jdk_valid_choices,
self.custom_jdk_number) = self._populate_jdk_configs(
properties, jdk_num)
jdk_num = get_validated_string_input(jdk_choice_prompt, jdk_num,
jdk_valid_choices,
"Invalid number.", False)
self.jdk_index = int(jdk_num) - 1
if self.jdk_index == self.custom_jdk_number:
print_warning_msg(
"JDK must be installed on all hosts and JAVA_HOME must be valid on all hosts."
)
print_warning_msg(jcePolicyWarn)
if get_silent():
print_error_msg(
"Path to JAVA_HOME should be specified via -j option.")
sys.exit(1)
args.java_home = get_validated_string_input(
"Path to JAVA_HOME: ", None, None, None, False, False)
if not os.path.exists(args.java_home) or not os.path.isfile(
os.path.join(args.java_home, "bin", self.JAVA_BIN)):
err = "Java home path or java binary file is unavailable. Please put correct path to java home."
raise FatalException(1, err)
print "Validating JDK on Ambari Server...done."
properties.process_pair(JAVA_HOME_PROPERTY, args.java_home)
properties.removeOldProp(JDK_NAME_PROPERTY)
properties.removeOldProp(JCE_NAME_PROPERTY)
# Make sure any previously existing JDK and JCE name properties are removed. These will
# confuse things in a Custom JDK scenario
properties.removeProp(JDK_NAME_PROPERTY)
properties.removeProp(JCE_NAME_PROPERTY)
self._ensure_java_home_env_var_is_set(args.java_home)
return
jdk_cfg = self.jdks[self.jdk_index]
resources_dir = get_resources_location(properties)
dest_file = os.path.abspath(
os.path.join(resources_dir, jdk_cfg.dest_file))
if os.path.exists(dest_file):
print "JDK already exists, using " + dest_file
elif properties[JDK_DOWNLOAD_SUPPORTED_PROPERTY].upper() == "FALSE":
print "ERROR: Oracle JDK is not found in {1}. JDK download is not supported in this distribution. Please download Oracle JDK " \
"archive ({0}) manually from Oracle site, place it into {1} and re-run this script.".format(jdk_cfg.dest_file, dest_file)
print "NOTE: If you have already downloaded the file, please verify if the name is exactly same as {0}.".format(
jdk_cfg.dest_file)
print 'Exiting...'
sys.exit(1)
else:
ok = get_YN_input(
"To download the Oracle JDK and the Java Cryptography Extension (JCE) "
"Policy Files you must accept the "
"license terms found at "
"http://www.oracle.com/technetwork/java/javase/"
"terms/license/index.html and not accepting will "
"cancel the Ambari Server setup and you must install the JDK and JCE "
"files manually.\nDo you accept the "
"Oracle Binary Code License Agreement [y/n] (y)? ", True)
if not ok:
print 'Exiting...'
sys.exit(1)
jdk_url = jdk_cfg.url
print 'Downloading JDK from ' + jdk_url + ' to ' + dest_file
self._download_jdk(jdk_url, dest_file, progress_func)
try:
(retcode, out,
java_home_dir) = self._install_jdk(dest_file, jdk_cfg)
except Exception, e:
print "Installation of JDK has failed: %s\n" % str(e)
file_exists = os.path.isfile(dest_file)
if file_exists:
ok = get_YN_input(
"JDK found at " + dest_file + ". "
"Would you like to re-download the JDK [y/n] (y)? ",
not get_silent())
if not ok:
err = "Unable to install JDK. Please remove JDK file found at " + \
dest_file + " and re-run Ambari Server setup"
raise FatalException(1, err)
else:
jdk_url = jdk_cfg.url
print 'Re-downloading JDK from ' + jdk_url + ' to ' + dest_file
self._download_jdk(jdk_url, dest_file, progress_func)
print 'Successfully re-downloaded JDK distribution to ' + dest_file
try:
(retcode, out) = self._install_jdk(dest_file, jdk_cfg)
except Exception, e:
print "Installation of JDK was failed: %s\n" % str(e)
err = "Unable to install JDK. Please remove JDK, file found at " + \
dest_file + " and re-run Ambari Server setup"
raise FatalException(1, err)
def _create_custom_user(self):
user = get_validated_string_input(
"Enter user account for ambari-server service ({0}):".format(
self.user), self.user, None, "Invalid username.", False)
if user in self.NR_SYSTEM_USERS:
self.user = user
return 0
if get_silent():
password = self.password
else:
password = get_validated_string_input(
"Enter password for user {0}:".format(user), "", None,
"Password", True, False)
from ambari_commons.os_windows import UserHelper
uh = UserHelper(user)
if uh.find_user():
print_info_msg(
"User {0} already exists, make sure that you typed correct password for user, "
"skipping user creation".format(user))
else:
status, message = uh.create_user(password)
if status == UserHelper.USER_EXISTS:
print_info_msg(
"User {0} already exists, make sure that you typed correct password for user, "
"skipping user creation".format(user))
elif status == UserHelper.ACTION_FAILED: # fail
print_warning_msg(
"Can't create user {0}. Failed with message {1}".format(
user, message))
return UserHelper.ACTION_FAILED
self.password = password
# setting SeServiceLogonRight and SeBatchLogonRight to user
#This is unconditional
status, message = uh.add_user_privilege('SeServiceLogonRight')
if status == UserHelper.ACTION_FAILED:
print_warning_msg(
"Can't add SeServiceLogonRight to user {0}. Failed with message {1}"
.format(user, message))
return UserHelper.ACTION_FAILED
status, message = uh.add_user_privilege('SeBatchLogonRight')
if status == UserHelper.ACTION_FAILED:
print_warning_msg(
"Can't add SeBatchLogonRight to user {0}. Failed with message {1}"
.format(user, message))
return UserHelper.ACTION_FAILED
print_info_msg("User configuration is done.")
print_warning_msg(
"When using non SYSTEM user make sure that your user has read\write access to log directories and "
"all server directories. In case of integrated authentication for SQL Server make sure that your "
"user is properly configured to access the ambari database.")
if user.find('\\') == -1:
user = '******' + user
self.user = user
return 0
