def extract_views():
java_exe_path = get_java_exe_path()
if java_exe_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
vdir = get_value_from_properties(properties, VIEWS_DIR_PROPERTY, configDefaults.DEFAULT_VIEWS_DIR)
files = [f for f in os.listdir(vdir) if os.path.isfile(os.path.join(vdir,f))]
for f in files:
command = VIEW_EXTRACT_CMD.format(java_exe_path,
get_full_ambari_classpath(), os.path.join(vdir,f))
retcode, stdout, stderr = run_os_command(command)
if retcode == 0:
sys.stdout.write(f + "\n")
elif retcode == 2:
sys.stdout.write("Error extracting " + f + "\n")
else:
sys.stdout.write(".")
sys.stdout.flush()
print_info_msg("Return code from extraction of view archive " + f + ": " +
str(retcode))
sys.stdout.write("\n")
return 0
def read_passwd_for_alias(alias, masterKey=""):
if alias:
jdk_path = find_jdk()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
tempFileName = "ambari.passwd"
passwd = ""
tempDir = tempfile.gettempdir()
#create temporary file for writing
tempFilePath = tempDir + os.sep + tempFileName
with open(tempFilePath, 'w+'):
os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE)
if masterKey is None or masterKey == "":
masterKey = "None"
command = SECURITY_PROVIDER_GET_CMD.format(get_java_exe_path(),
get_full_ambari_classpath(), alias, tempFilePath, masterKey)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider get passwd: " +
str(retcode))
if retcode != 0:
print 'ERROR: Unable to read password from store. alias = ' + alias
else:
with open(tempFilePath, 'r') as hfRTemp:
passwd = hfRTemp.read()
# Remove temporary file
os.remove(tempFilePath)
return passwd
else:
print_error_msg("Alias is unreadable.")
def get_conf_dir():
try:
conf_dir = os.environ[AMBARI_CONF_VAR]
return conf_dir
except KeyError:
default_conf_dir = configDefaults.DEFAULT_CONF_DIR
print_info_msg(AMBARI_CONF_VAR + " is not set, using default " + default_conf_dir)
return default_conf_dir
def get_ambari_jars():
try:
conf_dir = os.environ[AMBARI_SERVER_LIB]
return conf_dir
except KeyError:
default_jar_location = configDefaults.DEFAULT_LIBS_DIR
print_info_msg(AMBARI_SERVER_LIB + " is not set, using default "
+ default_jar_location)
return default_jar_location
def os_run_os_command(cmd, env=None, shell=False):
print_info_msg("about to run command: " + str(cmd))
if type(cmd) == str:
cmd = shlex.split(cmd)
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE, env=env, shell=shell
)
(stdoutdata, stderrdata) = process.communicate()
return process.returncode, stdoutdata, stderrdata
def _display_db_properties(self):
print_info_msg('Using database options: {database},{host},{port},{schema},{user},{password}'.format(
database=self.dbms,
host=self.database_host,
port=self.database_port,
schema=self.database_name,
user=self.database_username,
password=self.database_password
))
def _execute_db_script(databaseHost, databaseScript):
dbCmd = 'sqlcmd -S {0} -i {1}'.format(databaseHost, databaseScript)
retCode, outData, errData = run_os_command(['cmd', '/C', dbCmd])
if not retCode == 0:
err = 'Running database create script failed. Error output: {0} Output: {1} Exiting.'.format(errData, outData)
raise FatalException(retCode, err)
print_info_msg("sqlcmd output:")
print_info_msg(outData)
pass
def find_properties_file():
conf_file = search_file(AMS_PROPERTIES_FILE, get_conf_dir())
if conf_file is None:
err = 'File %s not found in search path $%s: %s' % (AMS_PROPERTIES_FILE,
AMS_CONF_VAR, get_conf_dir())
print err
raise FatalException(1, err)
else:
print_info_msg('Loading properties from ' + conf_file)
return conf_file
def store_password_file(password, filename):
conf_file = find_properties_file()
passFilePath = get_pass_file_path(conf_file, filename)
with open(passFilePath, 'w+') as passFile:
passFile.write(password)
print_info_msg("Adjusting filesystem permissions")
ambari_user = read_ambari_user()
set_file_permissions(passFilePath, "660", ambari_user, False)
#Windows paths need double backslashes, otherwise the Ambari server deserializer will think the single \ are escape markers
return passFilePath.replace('\\', '\\\\')
def _StopOrWaitForChildProcessToFinish(self, childProcess):
#Wait for the child process to finish or for the stop event to be signaled
if(win32event.WAIT_OBJECT_0 == win32event.WaitForMultipleObjects([WinService._heventSvcStop, childProcess._handle],
False, win32event.INFINITE)):
# The OS only detaches the child process when the master process exits.
# We must kill it manually.
try:
#Sending signal.CTRL_BREAK_EVENT doesn't work. It only detaches the child process from the master.
# Must brutally terminate the child process. Sorry Java.
childProcess.terminate()
except OSError, e:
print_info_msg("Unable to stop Ambari Server - " + str(e))
return False
def exec_ams_env_cmd(options):
ams_env_cmd = os.path.join(options.conf_dir, AMS_ENV_CMD)
if os.path.exists(ams_env_cmd):
cmds = ["cmd.exe", "/C", ams_env_cmd]
procAms = subprocess.Popen(cmds, env=os.environ)
out, err = procAms.communicate()
if err is not None and err is not "":
print_warning_msg(AMS_ENV_CMD + " error output: " + err)
if out is not None and out is not "":
print_info_msg(AMS_ENV_CMD + " output: " + out)
else:
err = 'ERROR: Cannot execute "{0}"'.format(ams_env_cmd)
raise FatalException(1, err)
def save_master_key(master_key, key_location, persist=True):
if master_key:
jdk_path = find_jdk()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
command = SECURITY_PROVIDER_KEY_CMD.format(get_java_exe_path(),
get_full_ambari_classpath(), master_key, key_location, persist)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider save KEY: " +
str(retcode))
else:
print_error_msg("Master key cannot be None.")
def move_user_custom_actions():
print_info_msg('Moving *.py files from custom_actions to custom_actions/scripts')
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
try:
resources_dir = properties[RESOURCES_DIR_PROPERTY]
except (KeyError), e:
conf_file = properties.fileName
err = 'Property ' + str(e) + ' is not defined at ' + conf_file
print_error_msg(err)
raise FatalException(1, err)
def stop(args):
if (args != None):
args.exit_message = None
status, pid = is_server_runing()
if status:
try:
os.killpg(os.getpgid(pid), signal.SIGKILL)
except OSError, e:
print_info_msg("Unable to stop Ambari Server - " + str(e))
return
pid_file_path = os.path.join(configDefaults.PID_DIR, PID_NAME)
os.remove(pid_file_path)
print "Ambari Server stopped"
def _restart_postgres():
print "Restarting PostgreSQL"
process = subprocess.Popen(
PGConfig.PG_RESTART_CMD.split(" "), stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE
)
time.sleep(5)
result = process.poll()
if result is None:
print_info_msg("Killing restart PostgresSQL process")
process.kill()
pg_status, retcode, out, err = PGConfig._get_postgre_status()
# SUSE linux set status of stopped postgresql proc to unused
if pg_status == "unused" or pg_status == "stopped":
print_info_msg("PostgreSQL is stopped. Restarting ...")
retcode, out, err = run_os_command(PGConfig.PG_START_CMD)
return retcode, out, err
return 0, "", ""
def select_dbms(self, options):
try:
dbms_index = options.database_index
except AttributeError:
db_name = get_value_from_properties(get_ambari_properties(), JDBC_DATABASE_PROPERTY, "").strip().lower()
persistence_type = get_value_from_properties(get_ambari_properties(), PERSISTENCE_TYPE_PROPERTY, "").strip().lower()
if persistence_type == STORAGE_TYPE_LOCAL:
dbms_index = self.DBMS_KEYS_LIST.index("embedded")
elif db_name:
dbms_index = self.DBMS_KEYS_LIST.index(db_name)
else:
dbms_index = self._get_default_dbms_index(options)
if options.must_set_database_options:
n_dbms = 1
dbms_choice_prompt = "==============================================================================\n" \
"Choose one of the following options:\n"
dbms_choices = ''
for desc in self.DBMS_LIST:
if len(desc.storage_name) > 0:
dbms_storage = " ({0})".format(desc.storage_name)
else:
dbms_storage = ""
dbms_choice_prompt += self.DBMS_PROMPT_PATTERN.format(n_dbms, desc.dbms_name, dbms_storage)
dbms_choices += str(n_dbms)
n_dbms += 1
database_num = str(dbms_index + 1)
dbms_choice_prompt += self.DBMS_CHOICE_PROMPT_PATTERN.format(database_num)
dbms_valid_choices = self.JDK_VALID_CHOICES_PATTERN.format(dbms_choices)
database_num = get_validated_string_input(
dbms_choice_prompt,
database_num,
dbms_valid_choices,
"Invalid number.",
False
)
dbms_index = int(database_num) - 1
if dbms_index >= n_dbms:
print_info_msg('Unknown db option, default to {0} {1}.'.format(
self.DBMS_LIST[0].storage_name, self.DBMS_LIST[0].dbms_name))
dbms_index = 0
return dbms_index
def run_schema_upgrade():
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
print 'Upgrading database schema'
command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath())
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from schema upgrade command, retcode = " + str(retcode))
if retcode > 0:
print_error_msg("Error executing schema upgrade, please check the server logs.")
else:
print_info_msg('Schema upgrade completed')
return retcode
def adjust_directory_permissions(ambari_user):
properties = get_ambari_properties()
bootstrap_dir = os.path.abspath(get_value_from_properties(properties, BOOTSTRAP_DIR_PROPERTY))
print_info_msg("Cleaning bootstrap directory ({0}) contents...".format(bootstrap_dir))
shutil.rmtree(bootstrap_dir, True) #Ignore the non-existent dir error
#Protect against directories lingering around
del_attempts = 0
while os.path.exists(bootstrap_dir) and del_attempts < 100:
time.sleep(50)
del_attempts += 1
if not os.path.exists(bootstrap_dir):
try:
os.makedirs(bootstrap_dir)
except Exception, ex:
print_warning_msg("Failed recreating the bootstrap directory: {0}".format(str(ex)))
pass
def run_schema_upgrade(args):
db_title = get_db_type(get_ambari_properties()).title
confirm = get_YN_input("Ambari Server configured for %s. Confirm "
"you have made a backup of the Ambari Server database [y/n] (y)? " % db_title, True)
if not confirm:
print_error_msg("Database backup is not confirmed")
return 1
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
ensure_jdbc_driver_is_installed(args, get_ambari_properties())
print 'Upgrading database schema'
serverClassPath = ServerClassPath(get_ambari_properties(), args)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()
command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, class_path)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
environ = generate_env(args, ambari_user, current_user)
(retcode, stdout, stderr) = run_os_command(command, env=environ)
print_info_msg("Return code from schema upgrade command, retcode = " + str(retcode))
if stdout:
print "Console output from schema upgrade command:"
print stdout
print
if stderr:
print "Error output from schema upgrade command:"
print stderr
print
if retcode > 0:
print_error_msg("Error executing schema upgrade, please check the server logs.")
else:
print_info_msg('Schema upgrade completed')
return retcode
def run_metainfo_upgrade(keyValueMap=None):
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
retcode = 1
if keyValueMap:
command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath(),
'updateMetaInfo',
"'" + json.dumps(keyValueMap) + "'")
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode))
if retcode > 0:
print_error_msg("Error executing metainfo upgrade, please check the "
"server logs.")
return retcode
def save_passwd_for_alias(alias, passwd, masterKey=""):
if alias and passwd:
jdk_path = find_jdk()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
if masterKey is None or masterKey == "":
masterKey = "None"
command = SECURITY_PROVIDER_PUT_CMD.format(get_java_exe_path(),
get_full_ambari_classpath(), alias, passwd, masterKey)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider save passwd: " +
str(retcode))
return retcode
else:
print_error_msg("Alias or password is unreadable.")
def __init__(self, options, properties, storage_type):
super(OracleConfig, self).__init__(options, properties, storage_type)
# Init the database configuration data here, if any
self.dbms = "oracle"
self.dbms_full_name = "Oracle"
self.driver_class_name = "oracle.jdbc.driver.OracleDriver"
self.driver_file_name = "ojdbc6.jar"
self.driver_symlink_name = "oracle-jdbc-driver.jar"
self.database_storage_name = "Service"
if (hasattr(options, "sid_or_sname") and options.sid_or_sname == "sname") or (
hasattr(options, "jdbc_url") and options.jdbc_url and re.match(ORACLE_SNAME_PATTERN, options.jdbc_url)
):
print_info_msg("using SERVICE_NAME instead of SID for Oracle")
self.sid_or_sname = "service_name"
self.database_port = DBMSConfig._init_member_with_prop_default(
options, "database_port", properties, JDBC_PORT_PROPERTY, "1521"
)
self.database_url_pattern = "jdbc:oracle:thin:@{0}:{1}/{2}"
self.database_url_pattern_alt = "jdbc:oracle:thin:@{0}:{1}:{2}"
self.JDBC_DRIVER_INSTALL_MSG = (
"Before starting Ambari Server, "
"you must copy the {0} JDBC driver JAR file to {1}.".format(
self.dbms_full_name, configDefaults.JAVA_SHARE_PATH
)
)
self.init_script_file = "/var/lib/tbds-server/resources/Ambari-DDL-Oracle-CREATE.sql'"
self.drop_tables_script_file = "/var/lib/tbds-server/resources/Ambari-DDL-Oracle-DROP.sql"
self.client_tool_usage_pattern = "sqlplus {1}/{2} < {0}"
self.jdbc_extra_params = [
["oracle.net.CONNECT_TIMEOUT", "2000"], # socket level timeout
["oracle.net.READ_TIMEOUT", "2000"], # socket level timeout
["oracle.jdbc.ReadTimeout", "8000"], # query fetch timeout
]
def _change_db_files_owner(self):
database_name = self.database_name
new_owner = self.database_username
if '"' not in new_owner:
#wrap to allow old username "ambari-server", postgres only
new_owner = '\'"{0}"\''.format(new_owner)
pass
command = PGConfig.CHANGE_OWNER_COMMAND[:]
command[-1] = command[-1].format(database_name, 'ambari', new_owner)
retcode, stdout, stderr = run_os_command(command)
if not retcode == 0:
if get_verbose():
if stderr:
print_error_msg("stderr:\n" + stderr.strip())
if stdout:
print_error_msg("stdout:\n" + stdout.strip())
else:
print_info_msg('Fixed database objects owner')
return retcode
def configure_os_settings():
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
try:
conf_os_type = properties[OS_TYPE_PROPERTY]
if conf_os_type != '':
print_info_msg("os_type already set in the properties file")
return 0
except (KeyError):
print_error_msg("os_type is not set in the properties file. Setting it now.")
# to check server/agent compatibility
master_os_family = OS_FAMILY + OS_VERSION
# to check supported os_types
master_os_type = OS_TYPE + OS_VERSION
write_property(OS_FAMILY_PROPERTY, master_os_family)
write_property(OS_TYPE_PROPERTY, master_os_type)
return 0
def _create_custom_user(self):
user = get_validated_string_input(
"Enter user account for ambari-server daemon (root):",
self.user,
"^[a-z_][a-z0-9_-]{1,31}$",
"Invalid username.",
False
)
print_info_msg("Trying to create user {0}".format(user))
command = self.NR_USERADD_CMD.format(user, self.NR_USER_COMMENT)
retcode, out, err = run_os_command(command)
if retcode == 9: # 9 = username already in use
print_info_msg("User {0} already exists, "
"skipping user creation".format(user))
elif retcode != 0: # fail
print_warning_msg("Can't create user {0}. Command {1} "
"finished with {2}: \n{3}".format(user, command, retcode, err))
return retcode
print_info_msg("User configuration is done.")
self.user = user
return 0
def _create_custom_user(self):
user = get_validated_string_input_optional(
"Enter user account for tbds-server service ({0}):".format(self.NR_DEFAULT_USER),
self.NR_DEFAULT_USER, None,
"Invalid username.",
False,
SETUP_USE_DEFAULT
)
if user == self.NR_DEFAULT_USER:
return 0, user
password = get_validated_string_input_optional("Enter password for user {0}:".format(user), "", None, "Password", True,SETUP_USE_DEFAULT, False)
from ambari_commons.os_windows import UserHelper
uh = UserHelper()
status, message = uh.create_user(user,password)
if status == UserHelper.USER_EXISTS:
print_info_msg("User {0} already exists, make sure that you typed correct password for user, "
"skipping user creation".format(user))
elif status == UserHelper.ACTION_FAILED: # fail
print_warning_msg("Can't create user {0}. Failed with message {1}".format(user, message))
return UserHelper.ACTION_FAILED, None
# setting SeServiceLogonRight to user
status, message = uh.add_user_privilege(user, 'SeServiceLogonRight')
if status == UserHelper.ACTION_FAILED:
print_warning_msg("Can't add SeServiceLogonRight to user {0}. Failed with message {1}".format(user, message))
return UserHelper.ACTION_FAILED, None
print_info_msg("User configuration is done.")
print_warning_msg("When using non SYSTEM user make sure that your user have read\write access to log directories and "
"all server directories. In case of integrated authentication for SQL Server make sure that your "
"user properly configured to use tbds and metric database.")
#storing username and password in os.environ temporary to pass them to service
os.environ[SERVICE_USERNAME_KEY] = user
os.environ[SERVICE_PASSWORD_KEY] = password
return 0, user
def select_dbms(self, options):
try:
dbms_index = options.database_index
except AttributeError:
dbms_index = self._get_default_dbms_index(options)
if options.must_set_database_options:
n_dbms = 1
dbms_choice_prompt = (
"==============================================================================\n"
"Choose one of the following options:\n"
)
dbms_choices = ""
for desc in self.DBMS_LIST:
if len(desc.storage_name) > 0:
dbms_storage = " ({0})".format(desc.storage_name)
else:
dbms_storage = ""
dbms_choice_prompt += self.DBMS_PROMPT_PATTERN.format(n_dbms, desc.dbms_name, dbms_storage)
dbms_choices += str(n_dbms)
n_dbms += 1
database_num = str(dbms_index + 1)
dbms_choice_prompt += self.DBMS_CHOICE_PROMPT_PATTERN.format(database_num)
dbms_valid_choices = self.JDK_VALID_CHOICES_PATTERN.format(dbms_choices)
database_num = get_validated_string_input(
dbms_choice_prompt, database_num, dbms_valid_choices, "Invalid number.", False
)
dbms_index = int(database_num) - 1
if dbms_index >= n_dbms:
print_info_msg(
"Unknown db option, default to {0} {1}.".format(
self.DBMS_LIST[0].storage_name, self.DBMS_LIST[0].dbms_name
)
)
dbms_index = 0
return dbms_index
def run_stack_upgrade(stackName, stackVersion, repo_url, repo_url_os):
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
stackId = {}
stackId[stackName] = stackVersion
if repo_url is not None:
stackId['repo_url'] = repo_url
if repo_url_os is not None:
stackId['repo_url_os'] = repo_url_os
command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath(),
"updateStackId",
"'" + json.dumps(stackId) + "'")
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode))
if retcode > 0:
print_error_msg("Error executing stack upgrade, please check the server logs.")
return retcode
def _create_custom_user(self):
user = get_validated_string_input(
"Enter user account for ambari-server service ({0}):".format(self.user),
self.user, None,
"Invalid username.",
False
)
if user in self.NR_SYSTEM_USERS:
self.user = user
return 0
if get_silent():
password = self.password
else:
password = get_validated_string_input("Enter password for user {0}:".format(user), "", None, "Password", True, False)
from ambari_commons.os_windows import UserHelper
uh = UserHelper(user)
if uh.find_user():
print_info_msg("User {0} already exists, make sure that you typed correct password for user, "
"skipping user creation".format(user))
else:
status, message = uh.create_user(password)
if status == UserHelper.USER_EXISTS:
print_info_msg("User {0} already exists, make sure that you typed correct password for user, "
"skipping user creation".format(user))
elif status == UserHelper.ACTION_FAILED: # fail
print_warning_msg("Can't create user {0}. Failed with message {1}".format(user, message))
return UserHelper.ACTION_FAILED
self.password = password
# setting SeServiceLogonRight and SeBatchLogonRight to user
#This is unconditional
status, message = uh.add_user_privilege('SeServiceLogonRight')
if status == UserHelper.ACTION_FAILED:
print_warning_msg("Can't add SeServiceLogonRight to user {0}. Failed with message {1}".format(user, message))
return UserHelper.ACTION_FAILED
status, message = uh.add_user_privilege('SeBatchLogonRight')
if status == UserHelper.ACTION_FAILED:
print_warning_msg("Can't add SeBatchLogonRight to user {0}. Failed with message {1}".format(user, message))
return UserHelper.ACTION_FAILED
print_info_msg("User configuration is done.")
print_warning_msg("When using non SYSTEM user make sure that your user has read\write access to log directories and "
"all server directories. In case of integrated authentication for SQL Server make sure that your "
"user is properly configured to access the ambari database.")
if user.find('\\') == -1:
user = '******' + user
self.user = user
return 0
def purge_stacks_and_mpacks(purge_list, replay_mode=False):
"""
Purge all stacks and management packs
:param replay_mode: Flag to indicate if purging in replay mode
"""
# Get ambari mpacks config properties
stack_location, extension_location, service_definitions_location, mpacks_staging_location = get_mpack_properties(
)
print_info_msg("Purging existing stack definitions and management packs")
if not purge_list:
print_info_msg("Nothing to purge")
return
# Don't delete default stack_advisor.py (stacks/stack_advisor.py)
if STACK_DEFINITIONS_RESOURCE_NAME in purge_list and os.path.exists(
stack_location):
print_info_msg("Purging stack location: " + stack_location)
for file in sorted(os.listdir(stack_location)):
path = os.path.join(stack_location, file)
if (os.path.isdir(path)):
sudo.rmtree(path)
if SERVICE_DEFINITIONS_RESOURCE_NAME in purge_list and os.path.exists(
service_definitions_location):
print_info_msg("Purging service definitions location: " +
service_definitions_location)
sudo.rmtree(service_definitions_location)
# Don't purge mpacks staging directory in replay mode
if MPACKS_RESOURCE_NAME in purge_list and not replay_mode and os.path.exists(
mpacks_staging_location):
print_info_msg("Purging mpacks staging location: " +
mpacks_staging_location)
sudo.rmtree(mpacks_staging_location)
sudo.makedir(mpacks_staging_location, 0755)
def _reset_local_database(self):
#force reset if silent option provided
if get_silent():
default = "yes"
else:
default = "no"
# Run automatic reset only for embedded DB
okToRun = get_YN_input("Confirm server reset [yes/no]({0})? ".format(default), get_silent())
if not okToRun:
err = "Ambari Server 'reset' cancelled"
raise FatalException(1, err)
print "Resetting the Server database..."
dbname = self.database_name
filename = self.drop_tables_script_file
username = self.database_username
password = self.database_password
command = PGConfig.SETUP_DB_CMD[:]
command[-1] = command[-1].format(filename, username, password, dbname)
drop_retcode, drop_outdata, drop_errdata = run_os_command(command)
if not drop_retcode == 0:
raise FatalException(1, drop_errdata)
if drop_errdata and PGConfig.PG_ERROR_BLOCKED in drop_errdata:
raise FatalException(1, "Database is in use. Please, make sure all connections to the database are closed")
if drop_errdata and get_verbose():
print_warning_msg(drop_errdata)
print_info_msg("About to run database setup")
retcode, outdata, errdata = self._setup_db()
if errdata and get_verbose():
print_warning_msg(errdata)
if (errdata and 'ERROR' in errdata.upper()) or (drop_errdata and 'ERROR' in drop_errdata.upper()):
err = "Non critical error in DDL"
if not get_verbose():
err += ", use --verbose for more information"
raise NonFatalException(err)
def _create_custom_user(self):
user = get_validated_string_input(
"Enter user account for ambari-server service ({0}):".format(self.user),
self.user, None,
"Invalid username.",
False
)
if user in self.NR_SYSTEM_USERS:
self.user = user
return 0
if get_silent():
password = self.password
else:
password = get_validated_string_input("Enter password for user {0}:".format(user), "", None, "Password", True, False)
from ambari_commons.os_windows import UserHelper
uh = UserHelper(user)
if uh.find_user():
print_info_msg("User {0} already exists, make sure that you typed correct password for user, "
"skipping user creation".format(user))
else:
status, message = uh.create_user(password)
if status == UserHelper.USER_EXISTS:
print_info_msg("User {0} already exists, make sure that you typed correct password for user, "
"skipping user creation".format(user))
elif status == UserHelper.ACTION_FAILED: # fail
print_warning_msg("Can't create user {0}. Failed with message {1}".format(user, message))
return UserHelper.ACTION_FAILED
self.password = password
# setting SeServiceLogonRight and SeBatchLogonRight to user
#This is unconditional
status, message = uh.add_user_privilege('SeServiceLogonRight')
if status == UserHelper.ACTION_FAILED:
print_warning_msg("Can't add SeServiceLogonRight to user {0}. Failed with message {1}".format(user, message))
return UserHelper.ACTION_FAILED
status, message = uh.add_user_privilege('SeBatchLogonRight')
if status == UserHelper.ACTION_FAILED:
print_warning_msg("Can't add SeBatchLogonRight to user {0}. Failed with message {1}".format(user, message))
return UserHelper.ACTION_FAILED
print_info_msg("User configuration is done.")
print_warning_msg("When using non SYSTEM user make sure that your user has read\write access to log directories and "
"all server directories. In case of integrated authentication for SQL Server make sure that your "
"user is properly configured to access the ambari database.")
if user.find('\\') == -1:
user = '******' + user
self.user = user
return 0
def _reset_local_database(self):
#force reset if silent option provided
if get_silent():
default = "yes"
else:
default = "no"
# Run automatic reset only for embedded DB
okToRun = get_YN_input("Confirm server reset [yes/no]({0})? ".format(default), get_silent())
if not okToRun:
err = "Ambari Server 'reset' cancelled"
raise FatalException(1, err)
print "Resetting the Server database..."
dbname = self.database_name
filename = self.drop_tables_script_file
username = self.database_username
password = self.database_password
command = PGConfig.SETUP_DB_CMD[:]
command[-1] = command[-1].format(filename, username, password, dbname)
drop_retcode, drop_outdata, drop_errdata = run_os_command(command)
if not drop_retcode == 0:
raise FatalException(1, drop_errdata)
if drop_errdata and PGConfig.PG_ERROR_BLOCKED in drop_errdata:
raise FatalException(1, "Database is in use. Please, make sure all connections to the database are closed")
if drop_errdata and get_verbose():
print_warning_msg(drop_errdata)
print_info_msg("About to run database setup")
retcode, outdata, errdata = self._setup_db()
if errdata and get_verbose():
print_warning_msg(errdata)
if (errdata and 'ERROR' in errdata.upper()) or (drop_errdata and 'ERROR' in drop_errdata.upper()):
err = "Non critical error in DDL"
if not get_verbose():
err += ", use --verbose for more information"
raise NonFatalException(err)
def replay_mpack_logs():
"""
Replay mpack logs during ambari-server upgrade
"""
replay_log_file = get_replay_log_file()
if os.path.exists(replay_log_file):
with open(replay_log_file, "r") as f:
for replay_log in f:
replay_log = replay_log.strip()
print_info_msg("===========================================================================================")
print_info_msg("Executing Mpack Replay Log :")
print_info_msg(replay_log)
print_info_msg("===========================================================================================")
replay_options = _named_dict(ast.literal_eval(replay_log))
if replay_options.mpack_command == INSTALL_MPACK_ACTION:
install_mpack(replay_options, replay_mode=True)
elif replay_options.mpack_command == UPGRADE_MPACK_ACTION:
upgrade_mpack(replay_options, replay_mode=True)
else:
error_msg = "Invalid mpack command {0} in mpack replay log {1}!".format(replay_options.mpack_command, replay_log_file)
print_error_msg(error_msg)
raise FatalException(-1, error_msg)
else:
print_info_msg("No mpack replay logs found. Skipping replaying mpack commands")
def run_metainfo_upgrade(args, keyValueMap=None):
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg(
"No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
retcode = 1
if keyValueMap:
serverClassPath = ServerClassPath(get_ambari_properties(), args)
command = STACK_UPGRADE_HELPER_CMD.format(
jdk_path,
serverClassPath.get_full_ambari_classpath_escaped_for_shell(),
'updateMetaInfo', "'" + json.dumps(keyValueMap) + "'")
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from stack upgrade command, retcode = " +
str(retcode))
if retcode > 0:
print_error_msg(
"Error executing metainfo upgrade, please check the "
"server logs.")
return retcode
def read_passwd_for_alias(alias, masterKey=""):
if alias:
jdk_path = find_jdk()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
tempFileName = "ambari.passwd"
passwd = ""
tempDir = tempfile.gettempdir()
#create temporary file for writing
tempFilePath = tempDir + os.sep + tempFileName
with open(tempFilePath, 'w+'):
os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE)
if masterKey is None or masterKey == "":
masterKey = "None"
serverClassPath = ambari_server.serverClassPath.ServerClassPath(get_ambari_properties(), None)
command = SECURITY_PROVIDER_GET_CMD.format(get_java_exe_path(),
serverClassPath.get_full_ambari_classpath_escaped_for_shell(), alias, tempFilePath, masterKey)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider get passwd: " +
str(retcode))
if retcode != 0:
print 'ERROR: Unable to read password from store. alias = ' + alias
else:
with open(tempFilePath, 'r') as hfRTemp:
passwd = hfRTemp.read()
# Remove temporary file
os.remove(tempFilePath)
return passwd
else:
print_error_msg("Alias is unreadable.")
def __init__(self, options, properties, storage_type):
super(OracleConfig, self).__init__(options, properties, storage_type)
#Init the database configuration data here, if any
self.dbms = "oracle"
self.dbms_full_name = "Oracle"
self.driver_class_name = "oracle.jdbc.driver.OracleDriver"
self.driver_file_name = "ojdbc6.jar"
self.driver_symlink_name = "oracle-jdbc-driver.jar"
self.database_storage_name = "Service"
if (hasattr(options, 'sid_or_sname') and options.sid_or_sname == "sname") or \
(hasattr(options, 'jdbc_url') and options.jdbc_url and re.match(ORACLE_SNAME_PATTERN, options.jdbc_url)):
print_info_msg("using SERVICE_NAME instead of SID for Oracle")
self.sid_or_sname = "sname"
self.database_port = DBMSConfig._init_member_with_prop_default(options, "database_port",
properties, JDBC_PORT_PROPERTY, "1521")
self.database_url_pattern = "jdbc:oracle:thin:@{0}:{1}/{2}"
self.database_url_pattern_alt = "jdbc:oracle:thin:@{0}:{1}:{2}"
self.JDBC_DRIVER_INSTALL_MSG = 'Before starting Ambari Server, ' \
'you must copy the {0} JDBC driver JAR file to {1}.'.format(
self.dbms_full_name, configDefaults.JAVA_SHARE_PATH)
self.init_script_file = AmbariPath.get("/var/lib/ambari-server/resources/Ambari-DDL-Oracle-CREATE.sql'")
self.drop_tables_script_file = AmbariPath.get("/var/lib/ambari-server/resources/Ambari-DDL-Oracle-DROP.sql")
self.client_tool_usage_pattern = 'sqlplus {1}/{2} < {0}'
self.jdbc_extra_params = [
["oracle.net.CONNECT_TIMEOUT", "2000"], # socket level timeout
["oracle.net.READ_TIMEOUT", "2000"], # socket level timeout
["oracle.jdbc.ReadTimeout", "8000"] # query fetch timeout
]
def upgrade_local_repo(args):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
stack_location = get_stack_location(properties)
stack_root_local = os.path.join(stack_location, "HDPLocal")
if not os.path.exists(stack_root_local):
print_info_msg("HDPLocal stack directory does not exist, skipping")
return
stack_root = os.path.join(stack_location, "HDP")
if not os.path.exists(stack_root):
print_info_msg("HDP stack directory does not exist, skipping")
return
for stack_version_local in os.listdir(stack_root_local):
repo_file_local = os.path.join(stack_root_local, stack_version_local,
"repos", "repoinfo.xml.rpmsave")
if not os.path.exists(repo_file_local):
repo_file_local = os.path.join(stack_root_local,
stack_version_local, "repos",
"repoinfo.xml")
repo_file = os.path.join(stack_root, stack_version_local, "repos",
"repoinfo.xml")
print_info_msg("Local repo file: " + repo_file_local)
print_info_msg("Repo file: " + repo_file_local)
metainfo_update_items = {}
if os.path.exists(repo_file_local) and os.path.exists(repo_file):
local_values = load_stack_values(stack_version_local,
repo_file_local)
repo_values = load_stack_values(stack_version_local, repo_file)
for k, v in local_values.iteritems():
if repo_values.has_key(k):
local_url = local_values[k]
repo_url = repo_values[k]
if repo_url != local_url:
metainfo_update_items[k] = local_url
run_metainfo_upgrade(metainfo_update_items)
def switch_addon_services(config_file):
old_verbose_level = get_verbose()
set_verbose(True)
if not os.path.exists(config_file):
print_error_msg(
'Configuration file {0} does not exist!'.format(config_file))
set_verbose(old_verbose_level)
return 1
print_info_msg(
"Switching addon services using config file {0}".format(config_file))
stack_location, extension_location, service_definitions_location, mpacks_staging_location, dashboard_location = get_mpack_properties(
)
mpack_metadata = _named_dict(json.load(open(config_file, "r")))
if not mpack_metadata:
print_error_msg('Malformed configuration file {0}'.format(config_file))
set_verbose(old_verbose_level)
return 1
mpack_name = mpack_metadata.name
mpack_version = mpack_metadata.version
mpack_dirname = mpack_name + "-" + mpack_version
mpack_staging_dir = os.path.join(mpacks_staging_location, mpack_dirname)
options = _named_dict(
ast.literal_eval("{'force' : True, 'verbose' : True}"))
for artifact in mpack_metadata.artifacts:
# Artifact name (Friendly name)
artifact_name = artifact.name
# Artifact type (stack-definitions, extension-definitions, service-definitions, etc)
artifact_type = artifact.type
# Artifact directory with contents of the artifact
artifact_source_dir = os.path.join(mpack_staging_dir,
artifact.source_dir)
# Artifact directory with contents of the artifact
artifact_source_dir = os.path.join(mpack_staging_dir,
artifact.source_dir)
print_info_msg("Processing artifact {0} of type {1} in {2}".format(
artifact_name, artifact_type, artifact_source_dir))
if artifact.type == STACK_ADDON_SERVICE_DEFINITIONS_ARTIFACT_NAME:
process_stack_addon_service_definitions_artifact(
artifact, artifact_source_dir, options)
print_info_msg(
"Successfully switched addon services using config file {0}".format(
config_file))
set_verbose(old_verbose_level)
return 0
def purge_stacks_and_mpacks():
"""
Purge all stacks and management packs
"""
# Get ambari mpacks config properties
stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties(
)
print_info_msg("Purging existing stack definitions and management packs")
if os.path.exists(stack_location):
print_info_msg("Purging stack location: " + stack_location)
sudo.rmtree(stack_location)
if os.path.exists(service_definitions_location):
print_info_msg("Purging service definitions location: " +
service_definitions_location)
sudo.rmtree(service_definitions_location)
if os.path.exists(mpacks_staging_location):
print_info_msg("Purging mpacks staging location: " +
mpacks_staging_location)
sudo.rmtree(mpacks_staging_location)
sudo.makedir(mpacks_staging_location, 0755)
def uninstall_mpack(mpack_name, mpack_version):
"""
Uninstall specific management pack
:param mpack_name: Management pack name
:param mpack_version: Management pack version
"""
print_info_msg("Uninstalling management pack {0}-{1}".format(
mpack_name, mpack_version))
# Get ambari mpack properties
stack_location, extension_location, service_definitions_location, mpacks_staging_location = get_mpack_properties(
)
found = False
if os.path.exists(mpacks_staging_location) and os.path.isdir(
mpacks_staging_location):
staged_mpack_dirs = sorted(os.listdir(mpacks_staging_location))
for dir in staged_mpack_dirs:
if dir == MPACKS_CACHE_DIRNAME:
continue
staged_mpack_dir = os.path.join(mpacks_staging_location, dir)
if os.path.isdir(staged_mpack_dir):
staged_mpack_metadata = read_mpack_metadata(staged_mpack_dir)
if not staged_mpack_metadata:
print_error_msg(
"Skipping malformed management pack {0}-{1}. Metadata file missing!"
.format(staged_mpack_name, staged_mpack_version))
continue
staged_mpack_name = staged_mpack_metadata.name
staged_mpack_version = staged_mpack_metadata.version
if mpack_name == staged_mpack_name and compare_versions(
staged_mpack_version, mpack_version, format=True) == 0:
print_info_msg(
"Removing management pack staging location {0}".format(
staged_mpack_dir))
sudo.rmtree(staged_mpack_dir)
remove_symlinks(stack_location,
service_definitions_location,
staged_mpack_dir)
found = True
break
if not found:
print_error_msg("Management pack {0}-{1} is not installed!".format(
mpack_name, mpack_version))
else:
print_info_msg(
"Management pack {0}-{1} successfully uninstalled!".format(
mpack_name, mpack_version))
def _check_postgre_up():
pg_status, retcode, out, err = PGConfig._get_postgre_status()
if pg_status == PGConfig.PG_STATUS_RUNNING:
print_info_msg("PostgreSQL is running")
return pg_status, 0, out, err
else:
# run initdb only on non ubuntu systems as ubuntu does not have initdb cmd.
if not OSCheck.is_ubuntu_family():
print "Running initdb: This may take upto a minute."
retcode, out, err = run_os_command(PGConfig.PG_INITDB_CMD)
if retcode == 0:
print out
print "About to start PostgreSQL"
try:
process = subprocess.Popen(PGConfig.PG_START_CMD.split(' '),
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
if OSCheck.is_suse_family():
time.sleep(20)
result = process.poll()
print_info_msg("Result of postgres start cmd: " +
str(result))
if result is None:
process.kill()
pg_status, retcode, out, err = PGConfig._get_postgre_status(
)
else:
retcode = result
else:
out, err = process.communicate()
retcode = process.returncode
pg_status, retcode, out, err = PGConfig._get_postgre_status(
)
if pg_status == PGConfig.PG_STATUS_RUNNING:
print_info_msg("Postgres process is running. Returning...")
return pg_status, 0, out, err
except (Exception), e:
pg_status, retcode, out, err = PGConfig._get_postgre_status()
if pg_status == PGConfig.PG_STATUS_RUNNING:
return pg_status, 0, out, err
else:
print_error_msg("Postgres start failed. " + str(e))
return pg_status, retcode, out, err
def _create_custom_user(self):
user = get_validated_string_input(
"Enter user account for ambari-server daemon (root):", "root",
"^[a-z_][a-z0-9_-]{1,31}$", "Invalid username.", False)
print_info_msg("Trying to create user {0}".format(user))
command = self.NR_USERADD_CMD.format(user, self.NR_USER_COMMENT)
retcode, out, err = run_os_command(command)
if retcode == 9: # 9 = username already in use
print_info_msg("User {0} already exists, "
"skipping user creation".format(user))
elif retcode != 0: # fail
print_warning_msg("Can't create user {0}. Command {1} "
"finished with {2}: \n{3}".format(
user, command, retcode, err))
return retcode, None
print_info_msg("User configuration is done.")
return 0, user
def DefCtrlCHandler():
print_info_msg("Ctrl+C handler invoked. Stopping.")
win32event.SetEvent(WinService._heventSvcStop)
pass
def install_mpack(options):
"""
Install management pack
:param options: Command line options
"""
mpack_path = options.mpack_path
if not mpack_path:
print_error_msg("Management pack not specified!")
raise FatalException(-1, 'Management pack not specified!')
print_info_msg("Installing management pack {0}".format(mpack_path))
# Download management pack to a temp location
tmp_archive_path = download_mpack(mpack_path)
# Expand management pack in temp directory
tmp_root_dir = expand_mpack(tmp_archive_path)
# Read mpack metadata
mpack_metadata = read_mpack_metadata(tmp_root_dir)
if not mpack_metadata:
raise FatalException(
-1, 'Malformed management pack {0}. Metadata file missing!'.format(
mpack_path))
# Validate management pack prerequisites
validate_mpack_prerequisites(mpack_metadata)
# Purge previously installed stacks and management packs
if options.purge:
purge_stacks_and_mpacks()
# Get ambari mpack properties
stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties(
)
# Create directories
if not os.path.exists(stack_location):
sudo.makedir(stack_location, 0755)
if not os.path.exists(service_definitions_location):
sudo.makedir(service_definitions_location, 0755)
if not os.path.exists(mpacks_staging_location):
sudo.makedir(mpacks_staging_location, 0755)
# Stage management pack (Stage at /var/lib/ambari-server/resources/mpacks/mpack_name-mpack_version)
mpack_name = mpack_metadata.name
mpack_version = mpack_metadata.version
mpack_dirname = mpack_name + "-" + mpack_version
mpack_staging_dir = os.path.join(mpacks_staging_location, mpack_dirname)
print_info_msg(
"Stage management pack {0}-{1} to staging location {2}".format(
mpack_name, mpack_version, mpack_staging_dir))
if os.path.exists(mpack_staging_dir):
if options.force:
print_info_msg(
"Force removing previously installed management pack from {0}".
format(mpack_staging_dir))
sudo.rmtree(mpack_staging_dir)
else:
error_msg = "Management pack {0}-{1} already installed!".format(
mpack_name, mpack_version)
print_error_msg(error_msg)
raise FatalException(-1, error_msg)
shutil.move(tmp_root_dir, mpack_staging_dir)
# Process setup steps for all artifacts (stack-definitions, service-definitions, stack-extension-definitions)
# in the management pack
for artifact in mpack_metadata.artifacts:
# Artifact name (Friendly name)
artifact_name = artifact.name
# Artifact type (stack-definitions, service-definitions, stack-extension-definitions etc)
artifact_type = artifact.type
# Artifact directory with contents of the artifact
artifact_source_dir = os.path.join(mpack_staging_dir,
artifact.source_dir)
print_info_msg("Processing artifact {0} of type {1} in {2}".format(
artifact_name, artifact_type, artifact_source_dir))
if artifact.type == "stack-definitions":
process_stack_definitions_artifact(artifact, artifact_source_dir,
options)
elif artifact.type == "stack-definition":
process_stack_definition_artifact(artifact, artifact_source_dir,
options)
elif artifact.type == "service-definitions":
process_service_definitions_artifact(artifact, artifact_source_dir,
options)
elif artifact.type == "service-definition":
process_service_definition_artifact(artifact, artifact_source_dir,
options)
elif artifact.type == "stack-extension-definitions":
process_stack_extension_definitions_artifact(
artifact, artifact_source_dir, options)
elif artifact.type == "stack-extension-definition":
process_stack_extension_definition_artifact(
artifact, artifact_source_dir, options)
else:
print_info_msg("Unknown artifact {0} of type {1}".format(
artifact_name, artifact_type))
print_info_msg("Management pack {0}-{1} successfully installed!".format(
mpack_name, mpack_version))
return mpack_name, mpack_version, mpack_staging_dir
def update_host_names(args, options):
services_stopped = userInput.get_YN_input(
"Please, confirm Ambari services are stopped [y/n] (n)? ", False)
if not services_stopped:
print 'Exiting...'
sys.exit(1)
pending_commands = userInput.get_YN_input(
"Please, confirm there are no pending commands on cluster [y/n] (n)? ",
False)
if not pending_commands:
print 'Exiting...'
sys.exit(1)
db_backup_done = userInput.get_YN_input(
"Please, confirm you have made backup of the Ambari db [y/n] (n)? ",
False)
if not db_backup_done:
print 'Exiting...'
sys.exit(1)
status, pid = serverUtils.is_server_runing()
if status:
raise FatalException(1, "Ambari Server should be stopped")
try:
host_mapping_file_path = args[1]
except IndexError:
#host_mapping file is mandatory
raise FatalException(
1,
"Invalid number of host update arguments. Probably, you forgot to add json file with "
"host changes.")
if not os.path.isfile(host_mapping_file_path):
raise FatalException(1, "Invalid file path or file doesn't exist")
if not os.access(host_mapping_file_path, os.R_OK):
raise FatalException(1, "File is not readable")
jdk_path = serverConfiguration.get_java_exe_path()
if jdk_path is None:
print_error_msg(
"No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
sys.exit(1)
properties = serverConfiguration.get_ambari_properties()
serverConfiguration.parse_properties_file(options)
options.database_index = LINUX_DBMS_KEYS_LIST.index(
properties[JDBC_DATABASE_PROPERTY])
dbConfiguration.ensure_jdbc_driver_is_installed(
options, serverConfiguration.get_ambari_properties())
serverClassPath = ServerClassPath(
serverConfiguration.get_ambari_properties(), options)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()
command = HOST_UPDATE_HELPER_CMD.format(jdk_path, class_path,
host_mapping_file_path)
ambari_user = serverConfiguration.read_ambari_user()
current_user = setupSecurity.ensure_can_start_under_current_user(
ambari_user)
environ = setupSecurity.generate_env(options, ambari_user, current_user)
(retcode, stdout, stderr) = os_utils.run_os_command(command, env=environ)
print_info_msg("Return code from update host names command, retcode = " +
str(retcode))
if retcode > 0:
print_error_msg(
"Error executing update host names, please check the server logs.")
raise FatalException(1, 'Host names update failed.')
else:
print_info_msg('Host names update completed successfully')
def upgrade(args):
print_info_msg("Upgrade Ambari Server", True)
if not is_root():
err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT
raise FatalException(4, err)
print_info_msg(
'Updating Ambari Server properties in {0} ...'.format(
AMBARI_PROPERTIES_FILE), True)
retcode = update_ambari_properties()
if not retcode == 0:
err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
print_info_msg(
'Updating Ambari Server properties in {0} ...'.format(AMBARI_ENV_FILE),
True)
retcode = update_ambari_env()
if not retcode == 0:
err = AMBARI_ENV_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
retcode = update_krb_jaas_login_properties()
if retcode == -2:
pass # no changes done, let's be silent
elif retcode == 0:
print_info_msg("File {0} updated.".format(AMBARI_KRB_JAAS_LOGIN_FILE),
True)
elif not retcode == 0:
err = AMBARI_KRB_JAAS_LOGIN_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
restore_custom_services()
replay_mpack_logs()
try:
update_database_name_property(upgrade=True)
except FatalException:
return -1
# Ignore the server version & database options passed via command-line arguments
parse_properties_file(args)
#TODO check database version
change_objects_owner(args)
retcode = run_schema_upgrade(args)
if not retcode == 0:
print_error_msg(
"Ambari server upgrade failed. Please look at {0}, for more details."
.format(configDefaults.SERVER_LOG_FILE))
raise FatalException(11, 'Schema upgrade failed.')
user = read_ambari_user()
if user is None:
warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG
print_warning_msg(warn)
else:
adjust_directory_permissions(user)
# create jdbc symlinks if jdbc drivers are available in resources
check_jdbc_drivers(args)
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
# Move *.py files from custom_actions to custom_actions/scripts
# This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0
ambari_version = get_ambari_version(properties)
if ambari_version is None:
args.warnings.append(
"*.py files were not moved from custom_actions to custom_actions/scripts."
)
elif compare_versions(ambari_version, "2.0.0") == 0:
move_user_custom_actions()
# Move files installed by package to default views directory to a custom one
for views_dir in get_views_dir(properties):
root_views_dir = views_dir + "/../"
if os.path.samefile(root_views_dir, get_default_views_dir()):
continue
for file in glob.glob(get_default_views_dir() + '/*'):
shutil.move(file, root_views_dir)
# Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0
admin_views_dirs = get_admin_views_dir(properties)
for admin_views_dir in admin_views_dirs:
shutil.rmtree(admin_views_dir)
# Modify timestamp of views jars to current time
views_jars = get_views_jars(properties)
for views_jar in views_jars:
os.utime(views_jar, None)
# check if ambari is configured to use LDAP authentication
if properties.get_property(CLIENT_SECURITY) == "ldap":
args.warnings.append(
"LDAP authentication is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration."
)
# adding custom jdbc name and previous custom jdbc properties
# we need that to support new dynamic jdbc names for upgraded ambari
add_jdbc_properties(properties)
json_url = get_json_url_from_repo_file()
if json_url:
print "Ambari repo file contains latest json url {0}, updating stacks repoinfos with it...".format(
json_url)
properties = get_ambari_properties()
stack_root = get_stack_location(properties)
update_latest_in_repoinfos_for_stacks(stack_root, json_url)
else:
print "Ambari repo file doesn't contain latest json url, skipping repoinfos modification"
ambari_repo_file = get_ambari_repo_file_full_name()
if ambari_repo_file:
if (os.path.exists(ambari_repo_file)):
ambari_repo_file_owner = get_file_owner(ambari_repo_file)
configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((ambari_repo_file, "644", ambari_repo_file_owner, False))
print "Adjusting ambari-server permissions and ownership..."
for pack in configDefaults.NR_ADJUST_OWNERSHIP_LIST:
file = pack[0]
mod = pack[1]
user = pack[2].format(ambari_user)
recursive = pack[3]
print_info_msg("Setting file permissions: {0} {1} {2} {3}".format(file, mod, user, recursive))
set_file_permissions(file, mod, user, recursive)
for pack in configDefaults.NR_CHANGE_OWNERSHIP_LIST:
path = pack[0]
user = pack[1].format(ambari_user)
recursive = pack[2]
print_info_msg("Changing ownership: {0} {1} {2}".format(path, user, recursive))
change_owner(path, user, recursive)
def configure_ldap_password():
passwordDefault = ""
passwordPrompt = 'Enter Manager Password* : '
passwordPattern = ".*"
passwordDescr = "Invalid characters in password."
def setup_sso(options):
print_info_msg("Setup SSO.")
server_status, pid = is_server_runing()
if not server_status:
err = 'Ambari Server is not running.'
raise FatalException(1, err)
if not get_silent():
validate_options(options)
ambari_properties = get_ambari_properties()
admin_login, admin_password = get_ambari_admin_username_password_pair(
options)
properties = get_sso_properties(ambari_properties, admin_login,
admin_password)
if not options.sso_enabled:
ambari_auth_enabled = get_value_from_dictionary(
properties, AMBARI_SSO_AUTH_ENABLED)
manage_services = get_value_from_dictionary(
properties, SSO_MANAGE_SERVICES)
if ambari_auth_enabled or manage_services:
if (ambari_auth_enabled and 'true' == ambari_auth_enabled) or \
(manage_services and 'true' == manage_services):
sso_status = "enabled"
else:
sso_status = "disabled"
else:
sso_status = "not configured"
sys.stdout.write("\nSSO is currently %s\n" % sso_status)
if sso_status == "enabled":
enable_sso = not get_YN_input(
"Do you want to disable SSO authentication [y/n] (n)? ",
False)
elif get_YN_input(
"Do you want to configure SSO authentication [y/n] (y)? ",
True):
enable_sso = True
else:
return False
else:
enable_sso = options.sso_enabled == 'true'
if enable_sso:
populate_sso_provider_url(options, properties)
populate_sso_public_cert(options, properties)
populate_ambari_requires_sso(options, properties)
populate_service_management(options, properties, ambari_properties,
admin_login, admin_password)
populate_jwt_cookie_name(options, properties)
populate_jwt_audiences(options, properties)
update_sso_conf(ambari_properties, properties, admin_login,
admin_password)
else:
remove_sso_conf(ambari_properties, admin_login, admin_password)
else:
warning = "setup-sso is not enabled in silent mode."
raise NonFatalException(warning)
pass
def _install_mpack(options, replay_mode=False, is_upgrade=False):
"""
Install management pack
:param options: Command line options
:param replay_mode: Flag to indicate if executing command in replay mode
"""
mpack_path = options.mpack_path
if not mpack_path:
print_error_msg("Management pack not specified!")
raise FatalException(-1, 'Management pack not specified!')
print_info_msg("Installing management pack {0}".format(mpack_path))
# Download management pack to a temp location
tmp_archive_path = download_mpack(mpack_path)
if not (tmp_archive_path and os.path.exists(tmp_archive_path)):
print_error_msg("Management pack could not be downloaded!")
raise FatalException(-1, 'Management pack could not be downloaded!')
# Expand management pack in temp directory
tmp_root_dir = expand_mpack(tmp_archive_path)
# Read mpack metadata
mpack_metadata = read_mpack_metadata(tmp_root_dir)
if not mpack_metadata:
raise FatalException(
-1, 'Malformed management pack {0}. Metadata file missing!'.format(
mpack_path))
# Validate management pack prerequisites
# Skip validation in replay mode
if not replay_mode:
validate_mpack_prerequisites(mpack_metadata)
if is_upgrade:
# Execute pre upgrade hook
_execute_hook(mpack_metadata, BEFORE_UPGRADE_HOOK_NAME, tmp_root_dir)
else:
# Execute pre install hook
_execute_hook(mpack_metadata, BEFORE_INSTALL_HOOK_NAME, tmp_root_dir)
# Purge previously installed stacks and management packs
if options.purge and options.purge_list:
purge_stacks_and_mpacks(options.purge_list.split(","), replay_mode)
# Get ambari mpack properties
stack_location, extension_location, service_definitions_location, mpacks_staging_location = get_mpack_properties(
)
mpacks_cache_location = os.path.join(mpacks_staging_location,
MPACKS_CACHE_DIRNAME)
# Create directories
if not os.path.exists(stack_location):
sudo.makedir(stack_location, 0755)
if not os.path.exists(extension_location):
sudo.makedir(extension_location, 0755)
if not os.path.exists(service_definitions_location):
sudo.makedir(service_definitions_location, 0755)
if not os.path.exists(mpacks_staging_location):
sudo.makedir(mpacks_staging_location, 0755)
if not os.path.exists(mpacks_cache_location):
sudo.makedir(mpacks_cache_location, 0755)
# Stage management pack (Stage at /var/lib/ambari-server/resources/mpacks/mpack_name-mpack_version)
mpack_name = mpack_metadata.name
mpack_version = mpack_metadata.version
mpack_dirname = mpack_name + "-" + mpack_version
mpack_staging_dir = os.path.join(mpacks_staging_location, mpack_dirname)
mpack_archive_path = os.path.join(mpacks_cache_location,
os.path.basename(tmp_archive_path))
print_info_msg(
"Stage management pack {0}-{1} to staging location {2}".format(
mpack_name, mpack_version, mpack_staging_dir))
if os.path.exists(mpack_staging_dir):
if options.force:
print_info_msg(
"Force removing previously installed management pack from {0}".
format(mpack_staging_dir))
sudo.rmtree(mpack_staging_dir)
else:
error_msg = "Management pack {0}-{1} already installed!".format(
mpack_name, mpack_version)
print_error_msg(error_msg)
raise FatalException(-1, error_msg)
shutil.move(tmp_root_dir, mpack_staging_dir)
shutil.move(tmp_archive_path, mpack_archive_path)
# Process setup steps for all artifacts (stack-definitions, extension-definitions,
# service-definitions, stack-addon-service-definitions) in the management pack
for artifact in mpack_metadata.artifacts:
# Artifact name (Friendly name)
artifact_name = artifact.name
# Artifact type (stack-definitions, extension-definitions, service-definitions, etc)
artifact_type = artifact.type
# Artifact directory with contents of the artifact
artifact_source_dir = os.path.join(mpack_staging_dir,
artifact.source_dir)
print_info_msg("Processing artifact {0} of type {1} in {2}".format(
artifact_name, artifact_type, artifact_source_dir))
if artifact.type == STACK_DEFINITIONS_ARTIFACT_NAME:
process_stack_definitions_artifact(artifact, artifact_source_dir,
options)
elif artifact.type == EXTENSION_DEFINITIONS_ARTIFACT_NAME:
process_extension_definitions_artifact(artifact,
artifact_source_dir,
options)
elif artifact.type == SERVICE_DEFINITIONS_ARTIFACT_NAME:
process_service_definitions_artifact(artifact, artifact_source_dir,
options)
elif artifact.type == STACK_ADDON_SERVICE_DEFINITIONS_ARTIFACT_NAME:
process_stack_addon_service_definitions_artifact(
artifact, artifact_source_dir, options)
else:
print_info_msg("Unknown artifact {0} of type {1}".format(
artifact_name, artifact_type))
print_info_msg("Management pack {0}-{1} successfully installed!".format(
mpack_name, mpack_version))
return mpack_metadata, mpack_name, mpack_version, mpack_staging_dir, mpack_archive_path
def set_file_permissions(file, mod, user, recursive):
if os.path.exists(file):
os_set_file_permissions(file, mod, recursive, user)
else:
print_info_msg("File %s does not exist" % file)
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
try:
resources_dir = properties[RESOURCES_DIR_PROPERTY]
except (KeyError), e:
conf_file = properties.fileName
err = 'Property ' + str(e) + ' is not defined at ' + conf_file
print_error_msg(err)
raise FatalException(1, err)
custom_actions_dir_path = os.path.join(resources_dir, 'custom_actions')
custom_actions_scripts_dir_path = os.path.join(custom_actions_dir_path,
'scripts')
print_info_msg('Moving *.py files from %s to %s' %
(custom_actions_dir_path, custom_actions_scripts_dir_path))
try:
for custom_action_file_name in os.listdir(custom_actions_dir_path):
custom_action_file_path = os.path.join(custom_actions_dir_path,
custom_action_file_name)
if os.path.isfile(custom_action_file_path
) and custom_action_file_path.endswith('.py'):
print_info_msg(
'Moving %s to %s' %
(custom_action_file_path, custom_actions_scripts_dir_path))
shutil.move(custom_action_file_path,
custom_actions_scripts_dir_path)
except (OSError, shutil.Error) as e:
err = 'Upgrade failed. Can not move *.py files from %s to %s. ' % (
custom_actions_dir_path, custom_actions_scripts_dir_path) + str(e)
def server_process_main(options, scmStatus=None):
if scmStatus is not None:
scmStatus.reportStartPending()
# debug mode
try:
global DEBUG_MODE
DEBUG_MODE = options.debug
except AttributeError:
pass
# stop Java process at startup?
try:
global SUSPEND_START_MODE
SUSPEND_START_MODE = options.suspend_start
except AttributeError:
pass
#options.conf_dir <= --config
if not os.path.isdir(options.conf_dir):
err = 'ERROR: Cannot find configuration directory "{0}"'.format(
options.conf_dir)
raise FatalException(1, err)
#execute ams-env.cmd
exec_ams_env_cmd(options)
#Ensure the 3 Hadoop services required are started on the local machine
if not options.no_embedded_hbase:
from amc_service import ensure_hadoop_service_soft_dependencies
ensure_hadoop_service_soft_dependencies()
if scmStatus is not None:
scmStatus.reportStartPending()
java_exe = get_java_exe_path()
java_class_path = get_java_cp()
java_heap_max = build_jvm_args()
command_base = SERVER_START_CMD_DEBUG if (
DEBUG_MODE or SERVER_START_DEBUG) else SERVER_START_CMD
suspend_mode = 'y' if SUSPEND_START_MODE else 'n'
command = command_base.format(java_class_path, java_heap_max, suspend_mode)
if not os.path.exists(PID_DIR):
os.makedirs(PID_DIR, 0755)
#Ignore the requirement to run as root. In Windows, by default the child process inherits the security context
# and the environment from the parent process.
param_list = java_exe + " " + command
print_info_msg("Running server: " + str(param_list))
procJava = subprocess32.Popen(param_list, env=os.environ)
#wait for server process for SERVER_START_TIMEOUT seconds
print "Waiting for server start..."
pidJava = procJava.pid
if pidJava <= 0:
procJava.terminate()
exitcode = procJava.returncode
save_pid(exitcode, EXITCODE_OUT_FILE)
if scmStatus is not None:
scmStatus.reportStopPending()
raise FatalException(-1, AMC_DIE_MSG.format(exitcode, SERVER_OUT_FILE))
else:
save_pid(pidJava, PID_OUT_FILE)
print "Server PID at: " + PID_OUT_FILE
print "Server out at: " + SERVER_OUT_FILE
print "Server log at: " + SERVER_LOG_FILE
if scmStatus is not None:
scmStatus.reportStarted()
return procJava
def setup_sso(options):
logger.info("Setup SSO.")
if not is_root():
raise FatalException(
4,
'ambari-server setup-sso should be run with root-level privileges')
server_status, pid = is_server_runing()
if not server_status:
err = 'Ambari Server is not running.'
raise FatalException(1, err)
if not get_silent():
validate_options(options)
properties = get_ambari_properties()
admin_login, admin_password = get_ambari_admin_username_password_pair(
options)
if not options.sso_enabled:
sso_enabled_from_db = get_sso_property_from_db(
properties, admin_login, admin_password, SSO_MANAGE_SERVICES)
sso_enabled = sso_enabled_from_db == None or sso_enabled_from_db in [
'true'
]
print_info_msg(
"SSO is currently {0}".format(
"not configured" if sso_enabled_from_db == None else (
"enabled" if sso_enabled else "disabled")), True)
if sso_enabled:
enable_sso = not get_YN_input(
"Do you want to disable SSO authentication [y/n] (n)? ",
False)
else:
if get_YN_input(
"Do you want to configure SSO authentication [y/n] (y)? ",
True):
enable_sso = True
else:
return False
else:
enable_sso = options.sso_enabled == 'true'
services = ''
if enable_sso:
populate_sso_provider_url(options, properties)
populate_sso_public_cert(options, properties)
populate_jwt_cookie_name(options, properties)
populate_jwt_audiences(options, properties)
services = get_services_requires_sso(options, properties,
admin_login, admin_password)
update_sso_conf(properties, enable_sso, services, admin_login,
admin_password)
enable_jwt_auth = WILDCARD_FOR_ALL_SERVICES == services or SERVICE_NAME_AMBARI in services
properties.process_pair(JWT_AUTH_ENBABLED,
"true" if enable_jwt_auth else "false")
update_properties(properties)
pass
else:
warning = "setup-sso is not enabled in silent mode."
raise NonFatalException(warning)
pass
def upgrade(args):
print_info_msg("Upgrade Ambari Server", True)
if not is_root():
err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT
raise FatalException(4, err)
print_info_msg(
'Updating Ambari Server properties in {0} ...'.format(
AMBARI_PROPERTIES_FILE), True)
retcode = update_ambari_properties()
if not retcode == 0:
err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
print_info_msg(
'Updating Ambari Server properties in {0} ...'.format(AMBARI_ENV_FILE),
True)
retcode = update_ambari_env()
if not retcode == 0:
err = AMBARI_ENV_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
retcode = update_krb_jaas_login_properties()
if retcode == -2:
pass # no changes done, let's be silent
elif retcode == 0:
print_info_msg("File {0} updated.".format(AMBARI_KRB_JAAS_LOGIN_FILE),
True)
elif not retcode == 0:
err = AMBARI_KRB_JAAS_LOGIN_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
restore_custom_services()
replay_mpack_logs()
try:
update_database_name_property(upgrade=True)
except FatalException:
return -1
# Ignore the server version & database options passed via command-line arguments
parse_properties_file(args)
#TODO check database version
change_objects_owner(args)
retcode = run_schema_upgrade(args)
if not retcode == 0:
print_error_msg(
"Ambari server upgrade failed. Please look at {0}, for more details."
.format(configDefaults.SERVER_LOG_FILE))
raise FatalException(11, 'Schema upgrade failed.')
user = read_ambari_user()
if user is None:
warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG
print_warning_msg(warn)
else:
adjust_directory_permissions(user)
# local repo
upgrade_local_repo(args)
# create jdbc symlinks if jdbc drivers are available in resources
check_jdbc_drivers(args)
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
# Move *.py files from custom_actions to custom_actions/scripts
# This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0
ambari_version = get_ambari_version(properties)
if ambari_version is None:
args.warnings.append(
"*.py files were not moved from custom_actions to custom_actions/scripts."
)
elif compare_versions(ambari_version, "2.0.0") == 0:
move_user_custom_actions()
# Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0
admin_views_dirs = get_admin_views_dir(properties)
for admin_views_dir in admin_views_dirs:
shutil.rmtree(admin_views_dir)
# Modify timestamp of views jars to current time
views_jars = get_views_jars(properties)
for views_jar in views_jars:
os.utime(views_jar, None)
# check if ambari has obsolete LDAP configuration
if properties.get_property(
LDAP_PRIMARY_URL_PROPERTY
) and not properties.get_property(IS_LDAP_CONFIGURED):
args.warnings.append(
"Existing LDAP configuration is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration."
)
# adding custom jdbc name and previous custom jdbc properties
# we need that to support new dynamic jdbc names for upgraded ambari
add_jdbc_properties(properties)
def run_schema_upgrade(args):
db_title = get_db_type(get_ambari_properties()).title
confirm = get_YN_input(
"Ambari Server configured for %s. Confirm "
"you have made a backup of the Ambari Server database [y/n] (y)? " %
db_title, True)
if not confirm:
print_error_msg("Database backup is not confirmed")
return 1
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg(
"No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
ensure_jdbc_driver_is_installed(args, get_ambari_properties())
print_info_msg('Upgrading database schema', True)
serverClassPath = ServerClassPath(get_ambari_properties(), args)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell(
validate_classpath=True)
set_debug_mode_from_options(args)
debug_mode = get_debug_mode()
debug_start = (debug_mode & 1) or SCHEMA_UPGRADE_DEBUG
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
command = SCHEMA_UPGRADE_HELPER_CMD_DEBUG.format(
jdk_path, class_path,
suspend_mode) if debug_start else SCHEMA_UPGRADE_HELPER_CMD.format(
jdk_path, class_path)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
environ = generate_env(args, ambari_user, current_user)
(retcode, stdout, stderr) = run_os_command(command, env=environ)
upgrade_response = json.loads(stdout)
check_gpl_license_approved(upgrade_response)
print_info_msg(
"Return code from schema upgrade command, retcode = {0}".format(
str(retcode)), True)
if stdout:
print_info_msg("Console output from schema upgrade command:", True)
print_info_msg(stdout, True)
print
if retcode > 0:
print_error_msg(
"Error executing schema upgrade, please check the server logs.")
if stderr:
print_error_msg("Error output from schema upgrade command:")
print_error_msg(stderr)
print
else:
print_info_msg('Schema upgrade completed', True)
return retcode
# The launched shell process and sub-processes should have a group id that
# is different from the parent.
def make_process_independent():
if IS_FOREGROUND: # upstart script is not able to track process from different pgid.
return
processId = os.getpid()
if processId > 0:
try:
os.setpgid(processId, processId)
except OSError, e:
print_warning_msg('setpgid({0}, {0}) failed - {1}'.format(
pidJava, str(e)))
pass
print_info_msg("Running server: " + str(param_list))
procJava = subprocess.Popen(param_list,
env=environ,
preexec_fn=make_process_independent)
pidJava = procJava.pid
if pidJava <= 0:
procJava.terminate()
exitcode = procJava.returncode
exitfile = os.path.join(configDefaults.PID_DIR, EXITCODE_NAME)
save_pid(exitcode, exitfile)
if scmStatus is not None:
scmStatus.reportStopPending()
raise FatalException(
def server_process_main(options, scmStatus=None):
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
raise FatalException(-1, err)
properties_for_print = []
logger.info("Ambari server properties config:")
for key, value in properties.getPropertyDict().items():
if "passwd" not in key and "password" not in key:
properties_for_print.append(key + "=" + value)
logger.info(properties_for_print)
# debug mode, including stop Java process at startup
try:
set_debug_mode_from_options(options)
except AttributeError:
pass
if not check_reverse_lookup():
print_warning_msg(
"The hostname was not found in the reverse DNS lookup. "
"This may result in incorrect behavior. "
"Please check the DNS setup and fix the issue.")
check_database_name_property()
parse_properties_file(options)
is_active_instance = get_is_active_instance()
if not is_active_instance:
print_warning_msg(
"This instance of ambari server is not designated as active. Cannot start ambari server."
)
err = "This is not an active instance. Shutting down..."
raise FatalException(1, err)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
print_info_msg("Ambari Server is not running...")
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
if not options.skip_properties_validation:
missing_properties = get_missing_properties(properties)
if missing_properties:
err = "Required properties are not found: " + str(missing_properties) + ". To skip properties validation " \
"use \"--skip-properties-validation\""
raise FatalException(1, err)
# Preparations
if is_root():
print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT
ensure_jdbc_driver_is_installed(options, properties)
ensure_dbms_is_running(options, properties, scmStatus)
if scmStatus is not None:
scmStatus.reportStartPending()
refresh_stack_hash(properties)
if scmStatus is not None:
scmStatus.reportStartPending()
ensure_server_security_is_configured()
if scmStatus is not None:
scmStatus.reportStartPending()
java_exe = get_java_exe_path()
serverClassPath = ServerClassPath(properties, options)
debug_mode = get_debug_mode()
debug_start = (debug_mode & 1) or SERVER_START_DEBUG
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
environ = generate_env(options, ambari_user, current_user)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell(
validate_classpath=True)
if options.skip_database_check:
global jvm_args
jvm_args += " -DskipDatabaseConsistencyCheck"
print "Ambari Server is starting with the database consistency check skipped. Do not make any changes to your cluster " \
"topology or perform a cluster upgrade until you correct the database consistency issues. See \"" \
+ configDefaults.DB_CHECK_LOG + "\" for more details on the consistency issues."
properties.process_pair(CHECK_DATABASE_SKIPPED_PROPERTY, "true")
else:
print "Ambari database consistency check started..."
if options.fix_database_consistency:
jvm_args += " -DfixDatabaseConsistency"
properties.process_pair(CHECK_DATABASE_SKIPPED_PROPERTY, "false")
update_properties(properties)
param_list = generate_child_process_param_list(ambari_user, java_exe,
class_path, debug_start,
suspend_mode)
# The launched shell process and sub-processes should have a group id that
# is different from the parent.
def make_process_independent():
if IS_FOREGROUND: # upstart script is not able to track process from different pgid.
return
processId = os.getpid()
if processId > 0:
try:
os.setpgid(processId, processId)
except OSError, e:
print_warning_msg('setpgid({0}, {0}) failed - {1}'.format(
pidJava, str(e)))
pass
def server_process_main(options, scmStatus=None):
# debug mode, including stop Java process at startup
try:
set_debug_mode_from_options(options)
except AttributeError:
pass
if not check_reverse_lookup():
print_warning_msg(
"The hostname was not found in the reverse DNS lookup. "
"This may result in incorrect behavior. "
"Please check the DNS setup and fix the issue.")
check_database_name_property()
parse_properties_file(options)
is_active_instance = get_is_active_instance()
if not is_active_instance:
print_warning_msg(
"This instance of ambari server is not designated as active. Cannot start ambari server."
)
err = "This is not an active instance. Shutting down..."
raise FatalException(1, err)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
print_info_msg("Ambari Server is not running...")
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
properties = get_ambari_properties()
if not options.skip_properties_validation:
missing_properties = get_missing_properties(properties)
if missing_properties:
err = "Required properties are not found: " + str(missing_properties) + ". To skip properties validation " \
"use \"--skip-properties-validation\""
raise FatalException(1, err)
# Preparations
if is_root():
print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT
ensure_jdbc_driver_is_installed(options, properties)
ensure_dbms_is_running(options, properties, scmStatus)
if scmStatus is not None:
scmStatus.reportStartPending()
refresh_stack_hash(properties)
if scmStatus is not None:
scmStatus.reportStartPending()
ensure_server_security_is_configured()
if scmStatus is not None:
scmStatus.reportStartPending()
java_exe = get_java_exe_path()
serverClassPath = ServerClassPath(properties, options)
debug_mode = get_debug_mode()
debug_start = (debug_mode & 1) or SERVER_START_DEBUG
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
if options.skip_database_validation:
global jvm_args
jvm_args += " -DskipDatabaseConsistencyValidation"
param_list = generate_child_process_param_list(
ambari_user, java_exe,
serverClassPath.get_full_ambari_classpath_escaped_for_shell(
validate_classpath=True), debug_start, suspend_mode)
environ = generate_env(options, ambari_user, current_user)
if not os.path.exists(configDefaults.PID_DIR):
os.makedirs(configDefaults.PID_DIR, 0755)
# The launched shell process and sub-processes should have a group id that
# is different from the parent.
def make_process_independent():
if IS_FOREGROUND: # upstart script is not able to track process from different pgid.
return
processId = os.getpid()
if processId > 0:
try:
os.setpgid(processId, processId)
except OSError, e:
print_warning_msg('setpgid({0}, {0}) failed - {1}'.format(
pidJava, str(e)))
pass
def setup_master_key():
if not is_root():
err = 'Ambari-server setup should be run with ' \
'root-level privileges'
raise FatalException(4, err)
properties = get_ambari_properties()
if properties == -1:
raise FatalException(1, "Failed to read properties file.")
db_windows_auth_prop = properties.get_property(JDBC_USE_INTEGRATED_AUTH_PROPERTY)
db_sql_auth = False if db_windows_auth_prop and db_windows_auth_prop.lower() == 'true' else True
db_password = properties.get_property(JDBC_PASSWORD_PROPERTY)
# Encrypt passwords cannot be called before setup
if db_sql_auth and not db_password:
print 'Please call "setup" before "encrypt-passwords". Exiting...'
return 1
# Check configuration for location of master key
isSecure = get_is_secure(properties)
(isPersisted, masterKeyFile) = get_is_persisted(properties)
# Read clear text DB password from file
if db_sql_auth and not is_alias_string(db_password) and os.path.isfile(db_password):
with open(db_password, 'r') as passwdfile:
db_password = passwdfile.read()
ldap_password = properties.get_property(LDAP_MGR_PASSWORD_PROPERTY)
if ldap_password:
# Read clear text LDAP password from file
if not is_alias_string(ldap_password) and os.path.isfile(ldap_password):
with open(ldap_password, 'r') as passwdfile:
ldap_password = passwdfile.read()
ts_password = properties.get_property(SSL_TRUSTSTORE_PASSWORD_PROPERTY)
resetKey = False
masterKey = None
if isSecure:
print "Password encryption is enabled."
resetKey = get_YN_input("Do you want to reset Master Key? [y/n] (n): ", False)
# For encrypting of only unencrypted passwords without resetting the key ask
# for master key if not persisted.
if isSecure and not isPersisted and not resetKey:
print "Master Key not persisted."
masterKey = get_original_master_key(properties)
pass
# Make sure both passwords are clear-text if master key is lost
if resetKey:
if not isPersisted:
print "Master Key not persisted."
masterKey = get_original_master_key(properties)
# Unable get the right master key or skipped question <enter>
if not masterKey:
print "To disable encryption, do the following:"
print "- Edit " + find_properties_file() + \
" and set " + SECURITY_IS_ENCRYPTION_ENABLED + " = " + "false."
err = "{0} is already encrypted. Please call {1} to store unencrypted" \
" password and call 'encrypt-passwords' again."
if db_sql_auth and db_password and is_alias_string(db_password):
print err.format('- Database password', "'" + SETUP_ACTION + "'")
if ldap_password and is_alias_string(ldap_password):
print err.format('- LDAP manager password', "'" + LDAP_SETUP_ACTION + "'")
if ts_password and is_alias_string(ts_password):
print err.format('TrustStore password', "'" + LDAP_SETUP_ACTION + "'")
return 1
pass
pass
pass
# Read back any encrypted passwords
if db_sql_auth and db_password and is_alias_string(db_password):
db_password = read_passwd_for_alias(JDBC_RCA_PASSWORD_ALIAS, masterKey)
if ldap_password and is_alias_string(ldap_password):
ldap_password = read_passwd_for_alias(LDAP_MGR_PASSWORD_ALIAS, masterKey)
if ts_password and is_alias_string(ts_password):
ts_password = read_passwd_for_alias(SSL_TRUSTSTORE_PASSWORD_ALIAS, masterKey)
# Read master key, if non-secure or reset is true
if resetKey or not isSecure:
masterKey = read_master_key(resetKey)
persist = get_YN_input("Do you want to persist master key. If you choose " \
"not to persist, you need to provide the Master " \
"Key while starting the ambari server as an env " \
"variable named " + SECURITY_KEY_ENV_VAR_NAME + \
" or the start will prompt for the master key."
" Persist [y/n] (y)? ", True)
if persist:
save_master_key(masterKey, get_master_key_location(properties) + os.sep +
SECURITY_MASTER_KEY_FILENAME, persist)
elif not persist and masterKeyFile:
try:
os.remove(masterKeyFile)
print_info_msg("Deleting master key file at location: " + str(
masterKeyFile))
except Exception, e:
print 'ERROR: Could not remove master key file. %s' % e
# Blow up the credential store made with previous key, if any
store_file = get_credential_store_location(properties)
if os.path.exists(store_file):
try:
os.remove(store_file)
except:
print_warning_msg("Failed to remove credential store file.")
pass
pass
