def check_selinux():
try:
retcode, out, err = run_os_command(GET_SE_LINUX_ST_CMD)
se_status = re.search('(disabled|enabled)', out).group(0)
print "SELinux status is '" + se_status + "'"
if se_status == SE_STATUS_DISABLED:
return 0
else:
try:
se_mode = re.search('(enforcing|permissive)', out).group(0)
except AttributeError:
err = "Error determining SELinux mode. Exiting."
raise FatalException(1, err)
print "SELinux mode is '" + se_mode + "'"
if se_mode == SE_MODE_ENFORCING:
print "Temporarily disabling SELinux"
run_os_command(SE_SETENFORCE_CMD)
print_warning_msg(
"SELinux is set to 'permissive' mode and temporarily disabled.")
ok = get_YN_input("OK to continue [y/n] (y)? ", True)
if not ok:
raise FatalException(1, None)
return 0
except OSError:
print_warning_msg("Could not run {0}: OK".format(GET_SE_LINUX_ST_CMD))
return 0
def _install_jdk(self, java_inst_file, jdk_cfg):
jdk_inst_dir = jdk_cfg.inst_dir
print "Installing JDK to {0}".format(jdk_inst_dir)
retcode, out, err = run_os_command(self.CREATE_JDK_DIR_CMD.format(jdk_inst_dir))
retcode, out, err = run_os_command(self.CHMOD_JDK_DIR_CMD.format(jdk_inst_dir))
savedPath = os.getcwd()
os.chdir(jdk_inst_dir)
try:
if java_inst_file.endswith(".gz"):
retcode, out, err = run_os_command(self.UNTAR_JDK_ARCHIVE.format(java_inst_file))
else:
err = "JDK installation failed.Unknown file extension."
raise FatalException(1, err)
finally:
os.chdir(savedPath)
if retcode != 0:
err = "Installation of JDK returned exit code %s" % retcode
raise FatalException(retcode, err)
jdk_version = re.search(jdk_cfg.reg_exp, out).group(1)
java_home_dir = os.path.join(jdk_inst_dir, jdk_version)
print "Successfully installed JDK to {0}".format(jdk_inst_dir)
return (retcode, out, java_home_dir)
def _get_postgre_status():
retcode, out, err = run_os_command(PGConfig.PG_ST_CMD)
try:
pg_status = re.search('(unused|stopped|not running|inactive|running|active)', out, re.IGNORECASE).group(0).lower()
except AttributeError:
pg_status = None
return pg_status, retcode, out, err
def read_passwd_for_alias(alias, masterKey=""):
if alias:
jdk_path = find_jdk()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
tempFileName = "ambari.passwd"
passwd = ""
tempDir = tempfile.gettempdir()
#create temporary file for writing
tempFilePath = tempDir + os.sep + tempFileName
with open(tempFilePath, 'w+'):
os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE)
if masterKey is None or masterKey == "":
masterKey = "None"
command = SECURITY_PROVIDER_GET_CMD.format(get_java_exe_path(),
get_full_ambari_classpath(), alias, tempFilePath, masterKey)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider get passwd: " +
str(retcode))
if retcode != 0:
print 'ERROR: Unable to read password from store. alias = ' + alias
else:
with open(tempFilePath, 'r') as hfRTemp:
passwd = hfRTemp.read()
# Remove temporary file
os.remove(tempFilePath)
return passwd
else:
print_error_msg("Alias is unreadable.")
def _create_custom_user(self):
user = get_validated_string_input(
"Enter user account for ambari-server daemon (root):",
self.user,
"^[a-z_][a-z0-9_-]{1,31}$",
"Invalid username.",
False
)
print_info_msg("Trying to create user {0}".format(user))
command = self.NR_USERADD_CMD.format(user, self.NR_USER_COMMENT)
retcode, out, err = run_os_command(command)
if retcode == 9: # 9 = username already in use
print_info_msg("User {0} already exists, "
"skipping user creation".format(user))
elif retcode != 0: # fail
print_warning_msg("Can't create user {0}. Command {1} "
"finished with {2}: \n{3}".format(user, command, retcode, err))
return retcode
print_info_msg("User configuration is done.")
self.user = user
return 0
def extract_views():
java_exe_path = get_java_exe_path()
if java_exe_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
vdir = get_value_from_properties(properties, VIEWS_DIR_PROPERTY, configDefaults.DEFAULT_VIEWS_DIR)
files = [f for f in os.listdir(vdir) if os.path.isfile(os.path.join(vdir,f))]
for f in files:
command = VIEW_EXTRACT_CMD.format(java_exe_path,
get_full_ambari_classpath(), os.path.join(vdir,f))
retcode, stdout, stderr = run_os_command(command)
if retcode == 0:
sys.stdout.write(f + "\n")
elif retcode == 2:
sys.stdout.write("Error extracting " + f + "\n")
else:
sys.stdout.write(".")
sys.stdout.flush()
print_info_msg("Return code from extraction of view archive " + f + ": " +
str(retcode))
sys.stdout.write("\n")
return 0
def _setup_db(self):
#password access to ambari-server and mapred
dbname = self.database_name
scriptFile = PGConfig.POSTGRES_EMBEDDED_INIT_FILE
username = self.database_username
password = self.database_password
#setup DB
command = PGConfig.SETUP_DB_CMD[:]
command[-1] = command[-1].format(scriptFile, username, password, dbname)
for i in range(SETUP_DB_CONNECT_ATTEMPTS):
sys.stdout.write('Connecting to local database...')
retcode, outdata, errdata = run_os_command(command)
if retcode == 0:
print 'done.'
return retcode, outdata, errdata
timeOutMsg = 'connection timed out'
if (i+1) < SETUP_DB_CONNECT_ATTEMPTS:
timeOutMsg += '...retrying (%d)' % (i+1)
print timeOutMsg
time.sleep(SETUP_DB_CONNECT_TIMEOUT)
print 'unable to connect to database'
print_error_msg(errdata)
return retcode, outdata, errdata
def setup_component_https(component, command, property, alias):
if not get_silent():
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"tbds-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
properties = get_ambari_properties()
use_https = properties.get_property(property) in ['true']
if use_https:
if get_YN_input("Do you want to disable HTTPS for " + component + " [y/n] (n)? ", False):
truststore_path = get_truststore_path(properties)
truststore_password = get_truststore_password(properties)
run_component_https_cmd(get_delete_cert_command(jdk_path, alias, truststore_path, truststore_password))
properties.process_pair(property, "false")
else:
return
else:
if get_YN_input("Do you want to configure HTTPS for " + component + " [y/n] (y)? ", True):
truststore_type = get_truststore_type(properties)
truststore_path = get_truststore_path(properties)
truststore_password = get_truststore_password(properties)
run_os_command(get_delete_cert_command(jdk_path, alias, truststore_path, truststore_password))
import_cert_path = get_validated_filepath_input( \
"Enter path to " + component + " Certificate: ", \
"Certificate not found")
run_component_https_cmd(get_import_cert_command(jdk_path, alias, truststore_type, import_cert_path, truststore_path, truststore_password))
properties.process_pair(property, "true")
else:
return
conf_file = find_properties_file()
f = open(conf_file, 'w')
properties.store(f, "Changed by 'tbds-server " + command + "' command")
else:
print command + " is not enabled in silent mode."
def _execute_db_script(databaseHost, databaseScript):
dbCmd = 'sqlcmd -S {0} -i {1}'.format(databaseHost, databaseScript)
retCode, outData, errData = run_os_command(['cmd', '/C', dbCmd])
if not retCode == 0:
err = 'Running database create script failed. Error output: {0} Output: {1} Exiting.'.format(errData, outData)
raise FatalException(retCode, err)
print_info_msg("sqlcmd output:")
print_info_msg(outData)
pass
def _get_postgre_status():
retcode, out, err = run_os_command(PGConfig.PG_ST_CMD)
# on RHEL and SUSE PG_ST_COMD returns RC 0 for running and 3 for stoppped
if retcode == 0:
pg_status = PGConfig.PG_STATUS_RUNNING
else:
if retcode == 3:
pg_status = "stopped"
else:
pg_status = None
return pg_status, retcode, out, err
def _ensure_java_home_env_var_is_set(self, java_home_dir):
if not os.environ.has_key(JAVA_HOME) or os.environ[JAVA_HOME] != java_home_dir:
java_home_dir_unesc = compress_backslashes(java_home_dir)
retcode, out, err = run_os_command("SETX {0} {1} /M".format(JAVA_HOME, java_home_dir_unesc))
if retcode != 0:
print_warning_msg("SETX output: " + out)
print_warning_msg("SETX error output: " + err)
err = "Setting JAVA_HOME failed. Exit code={0}".format(retcode)
raise FatalException(1, err)
os.environ[JAVA_HOME] = java_home_dir
def _configure_pg_hba_ambaridb_users(conf_file, database_username):
with open(conf_file, "a") as pgHbaConf:
pgHbaConf.write("\n")
pgHbaConf.write("local all " + database_username + ",mapred md5")
pgHbaConf.write("\n")
pgHbaConf.write("host all " + database_username + ",mapred 0.0.0.0/0 md5")
pgHbaConf.write("\n")
pgHbaConf.write("host all " + database_username + ",mapred ::/0 md5")
pgHbaConf.write("\n")
retcode, out, err = run_os_command(PGConfig.PG_HBA_RELOAD_CMD)
if not retcode == 0:
raise FatalException(retcode, err)
def setup_truststore(import_cert=False):
if not get_silent():
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
properties = get_ambari_properties()
if get_YN_input("Do you want to configure a truststore [y/n] (y)? ", True):
truststore_type = get_truststore_type(properties)
truststore_path = get_truststore_path(properties)
truststore_password = get_truststore_password(properties)
if import_cert:
if get_YN_input("Do you want to import a certificate [y/n] (y)? ", True):
alias = get_validated_string_input("Please enter an alias for the certificate: ", "", None, None, False, False)
run_os_command(get_delete_cert_command(jdk_path, alias, truststore_path, truststore_password))
import_cert_path = get_validated_filepath_input( \
"Enter path to certificate: ", \
"Certificate not found")
run_component_https_cmd(get_import_cert_command(jdk_path, alias, truststore_type, import_cert_path, truststore_path, truststore_password))
else:
return
conf_file = find_properties_file()
f = open(conf_file, 'w')
properties.store(f, "Changed by 'ambari-server setup-security' command")
else:
print "setup-security is not enabled in silent mode."
def execOsCommand(osCommand, tries=1, try_sleep=0, ret=None, cwd=None):
ret = _ret_init(ret)
for i in range(0, tries):
if i > 0:
time.sleep(try_sleep)
_ret_append_stderr(ret, "Retrying " + str(osCommand))
retcode, stdout, stderr = run_os_command(osCommand, cwd=cwd)
_ret_merge(ret, retcode, stdout, stderr)
if retcode == 0:
break
return ret
def is_server_runing():
pid_file_path = os.path.join(configDefaults.PID_DIR, PID_NAME)
if os.path.exists(pid_file_path):
try:
f = open(pid_file_path, "r")
except IOError, ex:
raise FatalException(1, str(ex))
pid = f.readline().strip()
if not pid.isdigit():
err = "%s is corrupt. Removing" % (pid_file_path)
f.close()
run_os_command("rm -f " + pid_file_path)
raise NonFatalException(err)
f.close()
retcode, out, err = run_os_command("ps -p " + pid)
if retcode == 0:
return True, int(pid)
else:
return False, None
def execOsCommand(osCommand, tries=1, try_sleep=0, ret=None, cwd=None):
ret = _ret_init(ret)
for i in range(0, tries):
if i > 0:
time.sleep(try_sleep)
_ret_append_stderr(ret, "Retrying " + str(osCommand))
retcode, stdout, stderr = run_os_command(osCommand, cwd=cwd)
_ret_merge(ret, retcode, stdout, stderr)
if retcode == 0:
break
return ret
def run_schema_upgrade(args):
db_title = get_db_type(get_ambari_properties()).title
confirm = get_YN_input(
"Ambari Server configured for %s. Confirm "
"you have made a backup of the Ambari Server database [y/n] (y)? " %
db_title, True)
if not confirm:
print_error_msg("Database backup is not confirmed")
return 1
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg(
"No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
ensure_jdbc_driver_is_installed(args, get_ambari_properties())
print 'Upgrading database schema'
serverClassPath = ServerClassPath(get_ambari_properties(), args)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()
command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, class_path)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
environ = generate_env(args, ambari_user, current_user)
(retcode, stdout, stderr) = run_os_command(command, env=environ)
print_info_msg("Return code from schema upgrade command, retcode = " +
str(retcode))
if stdout:
print "Console output from schema upgrade command:"
print stdout
print
if stderr:
print "Error output from schema upgrade command:"
print stderr
print
if retcode > 0:
print_error_msg(
"Error executing schema upgrade, please check the server logs.")
else:
print_info_msg('Schema upgrade completed')
return retcode
def save_master_key(master_key, key_location, persist=True):
if master_key:
jdk_path = find_jdk()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
command = SECURITY_PROVIDER_KEY_CMD.format(get_java_exe_path(),
get_full_ambari_classpath(), master_key, key_location, persist)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider save KEY: " +
str(retcode))
else:
print_error_msg("Master key cannot be None.")
def get_unique_id_and_date():
if os_check.OSCheck.is_windows_family():
from ambari_commons.os_utils import run_os_command
code, out, err = run_os_command("cmd /c vol C:")
for line in out.splitlines():
if line.startswith(" Volume Serial Number is"):
id = line[25:]
else:
out = shell.checked_call("hostid")[1].split('\n')[-1] # bugfix: take the lastline (stdin is not tty part cut)
id = out.strip()
now = datetime.datetime.now()
date = now.strftime("%M%d%y")
return "id{id}_date{date}".format(id=id, date=date)
def is_server_runing():
pid_file_path = os.path.join(configDefaults.PID_DIR, PID_NAME)
if os.path.exists(pid_file_path):
try:
f = open(pid_file_path, "r")
except IOError, ex:
raise FatalException(1, str(ex))
pid = f.readline().strip()
if not pid.isdigit():
err = "'%s' is incorrect PID value. %s is corrupt. Removing" % (
pid, pid_file_path)
f.close()
run_os_command("rm -f " + pid_file_path)
raise NonFatalException(err)
f.close()
retcode, out, err = run_os_command("ps -p " + pid)
if retcode == 0:
return True, int(pid)
else:
return False, None
def _configure_pg_hba_ambaridb_users(conf_file, database_username):
with open(conf_file, "a") as pgHbaConf:
pgHbaConf.write("\n")
pgHbaConf.write("local all " + database_username +
",mapred md5")
pgHbaConf.write("\n")
pgHbaConf.write("host all " + database_username +
",mapred 0.0.0.0/0 md5")
pgHbaConf.write("\n")
pgHbaConf.write("host all " + database_username +
",mapred ::/0 md5")
pgHbaConf.write("\n")
retcode, out, err = run_os_command(PGConfig.PG_HBA_RELOAD_CMD)
if not retcode == 0:
raise FatalException(retcode, err)
def get_unique_id_and_date():
if os_check.OSCheck.is_windows_family():
from ambari_commons.os_utils import run_os_command
code, out, err = run_os_command("cmd /c vol C:")
for line in out.splitlines():
if line.startswith(" Volume Serial Number is"):
id = line[25:]
else:
out = shell.checked_call("hostid")[1].split('\n')[
-1] # bugfix: take the lastline (stdin is not tty part cut)
id = out.strip()
now = datetime.datetime.now()
date = now.strftime("%M%d%y")
return "id{id}_date{date}".format(id=id, date=date)
def check_database(options):
jdk_path = serverConfiguration.get_java_exe_path()
if jdk_path is None:
print_error_msg(
"No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
sys.exit(1)
properties = serverConfiguration.get_ambari_properties()
serverConfiguration.parse_properties_file(options)
database_type = properties[JDBC_DATABASE_PROPERTY]
if not database_type:
print_error_msg("Please run \"ambari-server setup\" command"
" to initialize ambari db properties.")
options.database_index = LINUX_DBMS_KEYS_LIST.index(
properties[JDBC_DATABASE_PROPERTY])
dbConfiguration.ensure_jdbc_driver_is_installed(
options, serverConfiguration.get_ambari_properties())
serverClassPath = ServerClassPath(
serverConfiguration.get_ambari_properties(), options)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()
command = CHECK_DATABASE_HELPER_CMD.format(jdk_path, class_path)
ambari_user = serverConfiguration.read_ambari_user()
current_user = setupSecurity.ensure_can_start_under_current_user(
ambari_user)
environ = setupSecurity.generate_env(options, ambari_user, current_user)
(retcode, stdout, stderr) = os_utils.run_os_command(command, env=environ)
print_info_msg("Return code from check database command, retcode = " +
str(retcode))
if retcode > 0:
print_error_msg(
"Database check failed to complete. Please check ambari-server.log and ambari-server-check-database.log for problem."
)
raise FatalException(1, 'Database check failed.')
else:
print str(stdout)
def _restart_postgres():
print "Restarting PostgreSQL"
process = subprocess.Popen(
PGConfig.PG_RESTART_CMD.split(" "), stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE
)
time.sleep(5)
result = process.poll()
if result is None:
print_info_msg("Killing restart PostgresSQL process")
process.kill()
pg_status, retcode, out, err = PGConfig._get_postgre_status()
# SUSE linux set status of stopped postgresql proc to unused
if pg_status == "unused" or pg_status == "stopped":
print_info_msg("PostgreSQL is stopped. Restarting ...")
retcode, out, err = run_os_command(PGConfig.PG_START_CMD)
return retcode, out, err
return 0, "", ""
def _restart_postgres():
print "Restarting PostgreSQL"
process = subprocess.Popen(PGConfig.PG_RESTART_CMD.split(' '),
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
time.sleep(5)
result = process.poll()
if result is None:
print_info_msg("Killing restart PostgresSQL process")
process.kill()
pg_status, retcode, out, err = PGConfig._get_postgre_status()
# SUSE linux set status of stopped postgresql proc to unused
if pg_status == "unused" or pg_status == "stopped":
print_info_msg("PostgreSQL is stopped. Restarting ...")
retcode, out, err = run_os_command(PGConfig.PG_START_CMD)
return retcode, out, err
return 0, "", ""
def run_schema_upgrade():
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
print 'Upgrading database schema'
command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath())
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from schema upgrade command, retcode = " + str(retcode))
if retcode > 0:
print_error_msg("Error executing schema upgrade, please check the server logs.")
else:
print_info_msg('Schema upgrade completed')
return retcode
def run_schema_upgrade():
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
print 'Upgrading database schema'
command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath())
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from schema upgrade command, retcode = " + str(retcode))
if retcode > 0:
print_error_msg("Error executing schema upgrade, please check the server logs.")
else:
print_info_msg('Schema upgrade completed')
return retcode
def run_schema_upgrade(args):
db_title = get_db_type(get_ambari_properties()).title
confirm = get_YN_input("Ambari Server configured for %s. Confirm "
"you have made a backup of the Ambari Server database [y/n] (y)? " % db_title, True)
if not confirm:
print_error_msg("Database backup is not confirmed")
return 1
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
ensure_jdbc_driver_is_installed(args, get_ambari_properties())
print 'Upgrading database schema'
serverClassPath = ServerClassPath(get_ambari_properties(), args)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()
command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, class_path)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
environ = generate_env(args, ambari_user, current_user)
(retcode, stdout, stderr) = run_os_command(command, env=environ)
print_info_msg("Return code from schema upgrade command, retcode = " + str(retcode))
if stdout:
print "Console output from schema upgrade command:"
print stdout
print
if stderr:
print "Error output from schema upgrade command:"
print stderr
print
if retcode > 0:
print_error_msg("Error executing schema upgrade, please check the server logs.")
else:
print_info_msg('Schema upgrade completed')
return retcode
def _check_postgre_up():
pg_status, retcode, out, err = PGConfig._get_postgre_status()
if pg_status == PGConfig.PG_STATUS_RUNNING:
print_info_msg("PostgreSQL is running")
return pg_status, 0, out, err
else:
# run initdb only on non ubuntu systems as ubuntu does not have initdb cmd.
if not OSCheck.is_ubuntu_family():
print "Running initdb: This may take upto a minute."
retcode, out, err = run_os_command(PGConfig.PG_INITDB_CMD)
if retcode == 0:
print out
print "About to start PostgreSQL"
try:
process = subprocess.Popen(PGConfig.PG_START_CMD.split(' '),
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
if OSCheck.is_suse_family():
time.sleep(20)
result = process.poll()
print_info_msg("Result of postgres start cmd: " +
str(result))
if result is None:
process.kill()
pg_status, retcode, out, err = PGConfig._get_postgre_status(
)
else:
retcode = result
else:
out, err = process.communicate()
retcode = process.returncode
pg_status, retcode, out, err = PGConfig._get_postgre_status(
)
if pg_status == PGConfig.PG_STATUS_RUNNING:
print_info_msg("Postgres process is running. Returning...")
return pg_status, 0, out, err
except (Exception), e:
pg_status, retcode, out, err = PGConfig._get_postgre_status()
if pg_status == PGConfig.PG_STATUS_RUNNING:
return pg_status, 0, out, err
else:
print_error_msg("Postgres start failed. " + str(e))
return pg_status, retcode, out, err
def get_cert_info(path):
retcode, out, err = run_os_command(GET_CRT_INFO_CMD.format(path))
if retcode != 0:
print 'Error getting Certificate info'
print err
return None
if out:
certInfolist = out.split(os.linesep)
else:
print 'Empty Certificate info'
return None
notBefore = None
notAfter = None
subject = None
for item in range(len(certInfolist)):
if certInfolist[item].startswith('notAfter='):
notAfter = certInfolist[item].split('=')[1]
if certInfolist[item].startswith('notBefore='):
notBefore = certInfolist[item].split('=')[1]
if certInfolist[item].startswith('subject='):
subject = certInfolist[item].split('=', 1)[1]
#Convert subj to dict
pattern = re.compile(r"[A-Z]{1,2}=[\w.-]{1,}")
if subject:
subjList = pattern.findall(subject)
keys = [item.split('=')[0] for item in subjList]
values = [item.split('=')[1] for item in subjList]
subjDict = dict(zip(keys, values))
result = subjDict
result['notBefore'] = notBefore
result['notAfter'] = notAfter
result['subject'] = subject
return result
else:
return {}
def get_cert_info(path):
retcode, out, err = run_os_command(GET_CRT_INFO_CMD.format(path))
if retcode != 0:
print 'Error getting Certificate info'
print err
return None
if out:
certInfolist = out.split(os.linesep)
else:
print 'Empty Certificate info'
return None
notBefore = None
notAfter = None
subject = None
for item in range(len(certInfolist)):
if certInfolist[item].startswith('notAfter='):
notAfter = certInfolist[item].split('=')[1]
if certInfolist[item].startswith('notBefore='):
notBefore = certInfolist[item].split('=')[1]
if certInfolist[item].startswith('subject='):
subject = certInfolist[item].split('=', 1)[1]
#Convert subj to dict
pattern = re.compile(r"[A-Z]{1,2}=[\w.-]{1,}")
if subject:
subjList = pattern.findall(subject)
keys = [item.split('=')[0] for item in subjList]
values = [item.split('=')[1] for item in subjList]
subjDict = dict(zip(keys, values))
result = subjDict
result['notBefore'] = notBefore
result['notAfter'] = notAfter
result['subject'] = subject
return result
else:
return {}
def check_database(options):
logger.info("Check database consistency.")
jdk_path = serverConfiguration.get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
sys.exit(1)
properties = serverConfiguration.get_ambari_properties()
serverConfiguration.parse_properties_file(options)
database_type = properties[JDBC_DATABASE_PROPERTY]
if not database_type:
print_error_msg("Please run \"ambari-server setup\" command"
" to initialize ambari db properties.")
sys.exit(1)
options.database_index = LINUX_DBMS_KEYS_LIST.index(properties[JDBC_DATABASE_PROPERTY])
dbConfiguration.ensure_jdbc_driver_is_installed(options, serverConfiguration.get_ambari_properties())
serverClassPath = ServerClassPath(serverConfiguration.get_ambari_properties(), options)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()
command = CHECK_DATABASE_HELPER_CMD.format(jdk_path, class_path)
ambari_user = serverConfiguration.read_ambari_user()
current_user = setupSecurity.ensure_can_start_under_current_user(ambari_user)
environ = setupSecurity.generate_env(options, ambari_user, current_user)
(retcode, stdout, stderr) = os_utils.run_os_command(command, env=environ)
if retcode > 0:
raise FatalException(int(retcode), "Database check failed to complete: {0}. \nPlease check {1} and {2} for more "
"information.".format(stdout+stderr, configDefaults.SERVER_LOG_FILE, configDefaults.DB_CHECK_LOG))
else:
print str(stdout)
if not stdout.startswith("No errors"):
print "Ambari Server 'check-database' completed"
sys.exit(1)
def run_metainfo_upgrade(keyValueMap=None):
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
retcode = 1
if keyValueMap:
command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath(),
'updateMetaInfo',
"'" + json.dumps(keyValueMap) + "'")
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode))
if retcode > 0:
print_error_msg("Error executing metainfo upgrade, please check the "
"server logs.")
return retcode
def _check_postgre_up():
pg_status, retcode, out, err = PGConfig._get_postgre_status()
if pg_status == PGConfig.PG_STATUS_RUNNING:
print_info_msg("PostgreSQL is running")
return pg_status, 0, out, err
else:
# run initdb only on non ubuntu systems as ubuntu does not have initdb cmd.
if OS_TYPE != OSConst.OS_UBUNTU:
print "Running initdb: This may take upto a minute."
retcode, out, err = run_os_command(PGConfig.PG_INITDB_CMD)
if retcode == 0:
print out
print "About to start PostgreSQL"
try:
process = subprocess.Popen(
PGConfig.PG_START_CMD.split(" "),
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if OS_TYPE == OSConst.OS_SUSE:
time.sleep(20)
result = process.poll()
print_info_msg("Result of postgres start cmd: " + str(result))
if result is None:
process.kill()
pg_status, retcode, out, err = PGConfig._get_postgre_status()
else:
retcode = result
else:
out, err = process.communicate()
retcode = process.returncode
pg_status, retcode, out, err = PGConfig._get_postgre_status()
if pg_status == PGConfig.PG_STATUS_RUNNING:
print_info_msg("Postgres process is running. Returning...")
return pg_status, 0, out, err
except (Exception), e:
pg_status, retcode, out, err = PGConfig._get_postgre_status()
if pg_status == PGConfig.PG_STATUS_RUNNING:
return pg_status, 0, out, err
else:
print_error_msg("Postgres start failed. " + str(e))
return pg_status, retcode, out, err
def _reset_local_database(self):
#force reset if silent option provided
if get_silent():
default = "yes"
else:
default = "no"
# Run automatic reset only for embedded DB
okToRun = get_YN_input(
"Confirm server reset [yes/no]({0})? ".format(default),
get_silent())
if not okToRun:
err = "Ambari Server 'reset' cancelled"
raise FatalException(1, err)
print "Resetting the Server database..."
dbname = self.database_name
filename = self.drop_tables_script_file
username = self.database_username
password = self.database_password
command = PGConfig.SETUP_DB_CMD[:]
command[-1] = command[-1].format(filename, username, password, dbname)
drop_retcode, drop_outdata, drop_errdata = run_os_command(command)
if not drop_retcode == 0:
raise FatalException(1, drop_errdata)
if drop_errdata and PGConfig.PG_ERROR_BLOCKED in drop_errdata:
raise FatalException(
1,
"Database is in use. Please, make sure all connections to the database are closed"
)
if drop_errdata and get_verbose():
print_warning_msg(drop_errdata)
print_info_msg("About to run database setup")
retcode, outdata, errdata = self._setup_db()
if errdata and get_verbose():
print_warning_msg(errdata)
if (errdata and 'ERROR' in errdata.upper()) or (
drop_errdata and 'ERROR' in drop_errdata.upper()):
err = "Non critical error in DDL"
if not get_verbose():
err += ", use --verbose for more information"
raise NonFatalException(err)
def run_metainfo_upgrade(keyValueMap=None):
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
retcode = 1
if keyValueMap:
command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath(),
'updateMetaInfo',
"'" + json.dumps(keyValueMap) + "'")
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode))
if retcode > 0:
print_error_msg("Error executing metainfo upgrade, please check the "
"server logs.")
return retcode
def extract_views(options):
java_exe_path = get_java_exe_path()
if java_exe_path is None:
print_error_msg(
"No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
vdir = get_value_from_properties(properties, VIEWS_DIR_PROPERTY,
configDefaults.DEFAULT_VIEWS_DIR)
files = [
f for f in os.listdir(vdir) if os.path.isfile(os.path.join(vdir, f))
]
serverClassPath = ServerClassPath(get_ambari_properties(), options)
for f in files:
command = VIEW_EXTRACT_CMD.format(
java_exe_path,
serverClassPath.get_full_ambari_classpath_escaped_for_shell(),
os.path.join(vdir, f))
retcode, stdout, stderr = run_os_command(command)
if retcode == 0:
sys.stdout.write(f + "\n")
# elif retcode == 2:
# sys.stdout.write("Error extracting " + f + "\n")
else:
sys.stdout.write(".")
sys.stdout.flush()
print_info_msg("Return code from extraction of view archive " + f +
": " + str(retcode))
sys.stdout.write("\n")
return 0
def save_passwd_for_alias(alias, passwd, masterKey=""):
if alias and passwd:
jdk_path = find_jdk()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
if masterKey is None or masterKey == "":
masterKey = "None"
command = SECURITY_PROVIDER_PUT_CMD.format(get_java_exe_path(),
get_full_ambari_classpath(), alias, passwd, masterKey)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider save passwd: " +
str(retcode))
return retcode
else:
print_error_msg("Alias or password is unreadable.")
def save_passwd_for_alias(alias, passwd, masterKey=""):
if alias and passwd:
jdk_path = find_jdk()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
if masterKey is None or masterKey == "":
masterKey = "None"
command = SECURITY_PROVIDER_PUT_CMD.format(get_java_exe_path(),
get_full_ambari_classpath(), alias, passwd, masterKey)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider save passwd: " +
str(retcode))
return retcode
else:
print_error_msg("Alias or password is unreadable.")
def _create_custom_user(self):
user = get_validated_string_input(
"Enter user account for ambari-server daemon (root):", "root",
"^[a-z_][a-z0-9_-]{1,31}$", "Invalid username.", False)
print_info_msg("Trying to create user {0}".format(user))
command = self.NR_USERADD_CMD.format(user, self.NR_USER_COMMENT)
retcode, out, err = run_os_command(command)
if retcode == 9: # 9 = username already in use
print_info_msg("User {0} already exists, "
"skipping user creation".format(user))
elif retcode != 0: # fail
print_warning_msg("Can't create user {0}. Command {1} "
"finished with {2}: \n{3}".format(
user, command, retcode, err))
return retcode, None
print_info_msg("User configuration is done.")
return 0, user
def _install_jdk(self, java_inst_file, jdk_cfg):
jdk_inst_dir = jdk_cfg.inst_dir
print "Installing JDK to {0}".format(jdk_inst_dir)
if not os.path.exists(jdk_inst_dir):
os.makedirs(jdk_inst_dir)
if java_inst_file.endswith(".exe"):
(dirname, filename) = os.path.split(java_inst_file)
installLogFilePath = os.path.join(configDefaults.OUT_DIR, filename + "-install.log")
#jre7u67.exe /s INSTALLDIR=<dir> STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-server\\jre7u67.exe-install.log
installCmd = [
java_inst_file,
"/s",
"INSTALLDIR=" + jdk_inst_dir,
"STATIC=1",
"WEB_JAVA=0",
"/L",
installLogFilePath
]
retcode, out, err = run_os_command(installCmd)
#TODO: support .msi file installations
#msiexec.exe jre.msi /s INSTALLDIR=<dir> STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-server\\jre7u67-install.log ?
else:
err = "JDK installation failed.Unknown file mask."
raise FatalException(1, err)
if retcode == 1603:
# JDK already installed
print "JDK already installed in {0}".format(jdk_inst_dir)
retcode = 0
else:
if retcode != 0:
err = "Installation of JDK returned exit code %s" % retcode
raise FatalException(retcode, err)
print "Successfully installed JDK to {0}".format(jdk_inst_dir)
# Don't forget to adjust the JAVA_HOME env var
return (retcode, out, jdk_inst_dir)
def _change_db_files_owner(self):
database_name = self.database_name
new_owner = self.database_username
if '"' not in new_owner:
#wrap to allow old username "ambari-server", postgres only
new_owner = '\'"{0}"\''.format(new_owner)
pass
command = PGConfig.CHANGE_OWNER_COMMAND[:]
command[-1] = command[-1].format(database_name, 'ambari', new_owner)
retcode, stdout, stderr = run_os_command(command)
if not retcode == 0:
if get_verbose():
if stderr:
print_error_msg("stderr:\n" + stderr.strip())
if stdout:
print_error_msg("stdout:\n" + stdout.strip())
else:
print_info_msg('Fixed database objects owner')
return retcode
def _change_db_files_owner(self):
database_name = self.database_name
new_owner = self.database_username
if '"' not in new_owner:
#wrap to allow old username "ambari-server", postgres only
new_owner = '\'"{0}"\''.format(new_owner)
pass
command = PGConfig.CHANGE_OWNER_COMMAND[:]
command[-1] = command[-1].format(database_name, 'ambari', new_owner)
retcode, stdout, stderr = run_os_command(command)
if not retcode == 0:
if get_verbose():
if stderr:
print_error_msg("stderr:\n" + stderr.strip())
if stdout:
print_error_msg("stdout:\n" + stdout.strip())
else:
print_info_msg('Fixed database objects owner')
return retcode
def read_passwd_for_alias(alias, masterKey=""):
if alias:
jdk_path = find_jdk()
if jdk_path is None:
print_error_msg(
"No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
tempFileName = "ambari.passwd"
passwd = ""
tempDir = tempfile.gettempdir()
#create temporary file for writing
tempFilePath = tempDir + os.sep + tempFileName
file = open(tempFilePath, 'w+')
os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE)
file.close()
if masterKey is None or masterKey == "":
masterKey = "None"
serverClassPath = ambari_server.serverClassPath.ServerClassPath(
get_ambari_properties(), None)
command = SECURITY_PROVIDER_GET_CMD.format(
get_java_exe_path(),
serverClassPath.get_full_ambari_classpath_escaped_for_shell(),
alias, tempFilePath, masterKey)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider get passwd: " +
str(retcode))
if retcode != 0:
print 'ERROR: Unable to read password from store. alias = ' + alias
else:
passwd = open(tempFilePath, 'r').read()
# Remove temporary file
os.remove(tempFilePath)
return passwd
else:
print_error_msg("Alias is unreadable.")
def run_stack_upgrade(stackName, stackVersion, repo_url, repo_url_os):
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
stackId = {}
stackId[stackName] = stackVersion
if repo_url is not None:
stackId['repo_url'] = repo_url
if repo_url_os is not None:
stackId['repo_url_os'] = repo_url_os
command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath(),
"updateStackId",
"'" + json.dumps(stackId) + "'")
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode))
if retcode > 0:
print_error_msg("Error executing stack upgrade, please check the server logs.")
return retcode
def run_stack_upgrade(stackName, stackVersion, repo_url, repo_url_os):
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
stackId = {}
stackId[stackName] = stackVersion
if repo_url is not None:
stackId['repo_url'] = repo_url
if repo_url_os is not None:
stackId['repo_url_os'] = repo_url_os
command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath(),
"updateStackId",
"'" + json.dumps(stackId) + "'")
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode))
if retcode > 0:
print_error_msg("Error executing stack upgrade, please check the server logs.")
return retcode
def run(self):
#os.environ['COMSPEC'] = 'c:\\System32\\WindowsPowerShell\\v1.0\\powershell.exe'
psrcommand = ["powershell.exe",
"-NoProfile",
"-InputFormat", "Text",
"-ExecutionPolicy", "unrestricted",
"-Command", self.command]
if self.params:
psrcommand.extend([self.params])
if DEBUG:
self.host_log.write("Running PowerShell command " + ' '.join(psrcommand))
self.host_log.write("==========================")
self.host_log.write("\nCommand start time " + datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
retcode, stdout, stderr = run_os_command(psrcommand)
errorMsg = stderr
if self.errorMessage and retcode != 0:
errorMsg = self.errorMessage + "\n" + stderr
log = stdout + "\n" + errorMsg
self.host_log.write(log)
self.host_log.write("PowerShell command execution finished")
self.host_log.write("host=" + self.host + ", exitcode=" + str(retcode))
self.host_log.write("Command end time " + datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
return {"exitstatus": retcode, "log": log, "errormsg": errorMsg}
def _reset_local_database(self):
#force reset if silent option provided
if get_silent():
default = "yes"
else:
default = "no"
# Run automatic reset only for embedded DB
okToRun = get_YN_input("Confirm server reset [yes/no]({0})? ".format(default), get_silent())
if not okToRun:
err = "Ambari Server 'reset' cancelled"
raise FatalException(1, err)
print "Resetting the Server database..."
dbname = self.database_name
filename = self.drop_tables_script_file
username = self.database_username
password = self.database_password
command = PGConfig.SETUP_DB_CMD[:]
command[-1] = command[-1].format(filename, username, password, dbname)
drop_retcode, drop_outdata, drop_errdata = run_os_command(command)
if not drop_retcode == 0:
raise FatalException(1, drop_errdata)
if drop_errdata and PGConfig.PG_ERROR_BLOCKED in drop_errdata:
raise FatalException(1, "Database is in use. Please, make sure all connections to the database are closed")
if drop_errdata and get_verbose():
print_warning_msg(drop_errdata)
print_info_msg("About to run database setup")
retcode, outdata, errdata = self._setup_db()
if errdata and get_verbose():
print_warning_msg(errdata)
if (errdata and 'ERROR' in errdata.upper()) or (drop_errdata and 'ERROR' in drop_errdata.upper()):
err = "Non critical error in DDL"
if not get_verbose():
err += ", use --verbose for more information"
raise NonFatalException(err)
def server_process_main(options, scmStatus=None):
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
raise FatalException(-1, err)
properties_for_print = []
logger.info("Ambari server properties config:")
for key, value in properties.getPropertyDict().items():
if "passwd" not in key and "password" not in key:
properties_for_print.append(key + "=" + value)
logger.info(properties_for_print)
# debug mode, including stop Java process at startup
try:
set_debug_mode_from_options(options)
except AttributeError:
pass
if not check_reverse_lookup():
print_warning_msg(
"The hostname was not found in the reverse DNS lookup. "
"This may result in incorrect behavior. "
"Please check the DNS setup and fix the issue.")
check_database_name_property()
parse_properties_file(options)
is_active_instance = get_is_active_instance()
if not is_active_instance:
print_warning_msg(
"This instance of ambari server is not designated as active. Cannot start ambari server."
)
err = "This is not an active instance. Shutting down..."
raise FatalException(1, err)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
print_info_msg("Ambari Server is not running...")
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
if not options.skip_properties_validation:
missing_properties = get_missing_properties(properties)
if missing_properties:
err = "Required properties are not found: " + str(missing_properties) + ". To skip properties validation " \
"use \"--skip-properties-validation\""
raise FatalException(1, err)
# Preparations
if is_root():
print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT
ensure_jdbc_driver_is_installed(options, properties)
ensure_dbms_is_running(options, properties, scmStatus)
if scmStatus is not None:
scmStatus.reportStartPending()
refresh_stack_hash(properties)
if scmStatus is not None:
scmStatus.reportStartPending()
ensure_server_security_is_configured()
if scmStatus is not None:
scmStatus.reportStartPending()
java_exe = get_java_exe_path()
serverClassPath = ServerClassPath(properties, options)
debug_mode = get_debug_mode()
debug_start = (debug_mode & 1) or SERVER_START_DEBUG
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
environ = generate_env(options, ambari_user, current_user)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell(
validate_classpath=True)
if options.skip_database_check:
global jvm_args
jvm_args += " -DskipDatabaseConsistencyCheck"
print "Ambari Server is starting with the database consistency check skipped. Do not make any changes to your cluster " \
"topology or perform a cluster upgrade until you correct the database consistency issues. See \"" \
+ configDefaults.DB_CHECK_LOG + "\" for more details on the consistency issues."
properties.process_pair(CHECK_DATABASE_SKIPPED_PROPERTY, "true")
else:
print "Ambari database consistency check started..."
properties.process_pair(CHECK_DATABASE_SKIPPED_PROPERTY, "false")
command = CHECK_DATABASE_HELPER_CMD.format(java_exe, class_path)
(retcode, stdout, stderr) = run_os_command(command, env=environ)
if retcode > 0:
print str(stdout)
raise FatalException(
1, 'Database check failed to complete. Please check ' +
configDefaults.SERVER_LOG_FILE + ' and ' +
configDefaults.DB_CHECK_LOG + ' for more information.')
else:
print str(stdout)
print "Ambari database consistency check finished"
if not stdout.startswith("No errors"):
sys.exit(1)
update_properties(properties)
param_list = generate_child_process_param_list(ambari_user, java_exe,
class_path, debug_start,
suspend_mode)
# The launched shell process and sub-processes should have a group id that
# is different from the parent.
def make_process_independent():
if IS_FOREGROUND: # upstart script is not able to track process from different pgid.
return
processId = os.getpid()
if processId > 0:
try:
os.setpgid(processId, processId)
except OSError, e:
print_warning_msg('setpgid({0}, {0}) failed - {1}'.format(
pidJava, str(e)))
pass
def run_component_https_cmd(cmd):
retcode, out, err = run_os_command(cmd)
if not retcode == 0:
err = 'Error occured during truststore setup ! :' + out + " : " + err
raise FatalException(1, err)
def run_db_cleanup(options):
if validate_args(options):
return 1
status, stateDesc = is_server_runing()
if not options.silent:
db_title = get_db_type(get_ambari_properties()).title
confirmBackup = get_YN_input(
"Ambari Server configured for {0}. Confirm you have made a backup of the Ambari Server database [y/n]"
.format(db_title), True)
if not confirmBackup:
print_info_msg("Ambari Server Database cleanup aborted")
return 0
if status:
print_error_msg(
"The database cleanup cannot proceed while Ambari Server is running. Please shut down Ambari first."
)
return 1
confirm = get_YN_input(
"Ambari server is using db type {0}. Cleanable database entries older than {1} will be cleaned up. Proceed [y/n]"
.format(db_title, options.cleanup_from_date), True)
if not confirm:
print_info_msg("Ambari Server Database cleanup aborted")
return 0
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg(
"No JDK found, please run the \"setup\" command to install a JDK automatically or install any "
"JDK manually to {0}".format(configDefaults.JDK_INSTALL_DIR))
return 1
ensure_jdbc_driver_is_installed(options, get_ambari_properties())
serverClassPath = ServerClassPath(get_ambari_properties(), options)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
environ = generate_env(options, ambari_user, current_user)
print "Cleaning up the database ..."
command = DB_CLEANUP_CMD.format(jdk_path, class_path, options.cluster_name,
options.cleanup_from_date)
(retcode, stdout, stderr) = run_os_command(command, env=environ)
print_info_msg("Return code from database cleanup command, retcode = " +
str(retcode))
if stdout:
print "Console output from database cleanup command:"
print stdout
print
if stderr:
print "Error output from database cleanup command:"
print stderr
print
if retcode > 0:
print_error_msg(
"Error wncountered while cleaning up the Ambari Server Database. Check the ambari-server.log for details."
)
else:
print "Cleanup completed. Check the ambari-server.log for details."
return retcode
def update_host_names(args, options):
services_stopped = userInput.get_YN_input(
"Please, confirm Ambari services are stopped [y/n] (n)? ", False)
if not services_stopped:
print 'Exiting...'
sys.exit(1)
pending_commands = userInput.get_YN_input(
"Please, confirm there are no pending commands on cluster [y/n] (n)? ",
False)
if not pending_commands:
print 'Exiting...'
sys.exit(1)
db_backup_done = userInput.get_YN_input(
"Please, confirm you have made backup of the Ambari db [y/n] (n)? ",
False)
if not db_backup_done:
print 'Exiting...'
sys.exit(1)
status, pid = serverUtils.is_server_runing()
if status:
raise FatalException(1, "Ambari Server should be stopped")
try:
host_mapping_file_path = args[1]
except IndexError:
#host_mapping file is mandatory
raise FatalException(
1,
"Invalid number of host update arguments. Probably, you forgot to add json file with "
"host changes.")
if not os.path.isfile(host_mapping_file_path):
raise FatalException(1, "Invalid file path or file doesn't exist")
if not os.access(host_mapping_file_path, os.R_OK):
raise FatalException(1, "File is not readable")
jdk_path = serverConfiguration.get_java_exe_path()
if jdk_path is None:
print_error_msg(
"No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
sys.exit(1)
properties = serverConfiguration.get_ambari_properties()
serverConfiguration.parse_properties_file(options)
options.database_index = LINUX_DBMS_KEYS_LIST.index(
properties[JDBC_DATABASE_PROPERTY])
dbConfiguration.ensure_jdbc_driver_is_installed(
options, serverConfiguration.get_ambari_properties())
serverClassPath = ServerClassPath(
serverConfiguration.get_ambari_properties(), options)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()
command = HOST_UPDATE_HELPER_CMD.format(jdk_path, class_path,
host_mapping_file_path)
ambari_user = serverConfiguration.read_ambari_user()
current_user = setupSecurity.ensure_can_start_under_current_user(
ambari_user)
environ = setupSecurity.generate_env(options, ambari_user, current_user)
(retcode, stdout, stderr) = os_utils.run_os_command(command, env=environ)
print_info_msg("Return code from update host names command, retcode = " +
str(retcode))
if retcode > 0:
print_error_msg(
"Error executing update host names, please check the server logs.")
raise FatalException(1, 'Host names update failed.')
else:
print_info_msg('Host names update completed successfully')
def import_cert_and_key(security_server_keys_dir):
import_cert_path = get_validated_filepath_input( \
"Enter path to Certificate: ", \
"Certificate not found")
import_key_path = get_validated_filepath_input( \
"Enter path to Private Key: ", "Private Key not found")
pem_password = get_validated_string_input(
"Please enter password for Private Key: ", "", None, None, True)
certInfoDict = get_cert_info(import_cert_path)
if not certInfoDict:
print_warning_msg('Unable to get Certificate information')
else:
#Validate common name of certificate
if not is_valid_cert_host(certInfoDict):
print_warning_msg('Unable to validate Certificate hostname')
#Validate issue and expirations dates of certificate
if not is_valid_cert_exp(certInfoDict):
print_warning_msg(
'Unable to validate Certificate issue and expiration dates')
#jetty requires private key files with non-empty key passwords
retcode = 0
err = ''
if not pem_password:
print 'Generating random password for HTTPS keystore...done.'
pem_password = generate_random_string()
retcode, out, err = run_os_command(
CHANGE_KEY_PWD_CND.format(import_key_path, pem_password))
import_key_path += '.secured'
if retcode == 0:
keystoreFilePath = os.path.join(security_server_keys_dir, \
SSL_KEYSTORE_FILE_NAME)
keystoreFilePathTmp = os.path.join(tempfile.gettempdir(), \
SSL_KEYSTORE_FILE_NAME)
passFilePath = os.path.join(security_server_keys_dir, \
SSL_KEY_PASSWORD_FILE_NAME)
passFilePathTmp = os.path.join(tempfile.gettempdir(), \
SSL_KEY_PASSWORD_FILE_NAME)
passinFilePath = os.path.join(tempfile.gettempdir(), \
SSL_PASSIN_FILE)
passwordFilePath = os.path.join(tempfile.gettempdir(), \
SSL_PASSWORD_FILE)
with open(passFilePathTmp, 'w+') as passFile:
passFile.write(pem_password)
passFile.close
pass
set_file_permissions(passFilePath, "660", read_ambari_user(), False)
copy_file(passFilePathTmp, passinFilePath)
copy_file(passFilePathTmp, passwordFilePath)
retcode, out, err = run_os_command(EXPRT_KSTR_CMD.format(import_cert_path, \
import_key_path, passwordFilePath, passinFilePath, keystoreFilePathTmp))
if retcode == 0:
print 'Importing and saving Certificate...done.'
import_file_to_keystore(keystoreFilePathTmp, keystoreFilePath)
import_file_to_keystore(passFilePathTmp, passFilePath)
import_file_to_keystore(import_cert_path, os.path.join( \
security_server_keys_dir, SSL_CERT_FILE_NAME))
import_file_to_keystore(import_key_path, os.path.join( \
security_server_keys_dir, SSL_KEY_FILE_NAME))
#Validate keystore
retcode, out, err = run_os_command(VALIDATE_KEYSTORE_CMD.format(keystoreFilePath, \
passwordFilePath, passinFilePath))
remove_file(passinFilePath)
remove_file(passwordFilePath)
if not retcode == 0:
print 'Error during keystore validation occured!:'
print err
return False
return True
else:
print_error_msg('Could not import Certificate and Private Key.')
print 'SSL error on exporting keystore: ' + err.rstrip() + \
'.\nPlease ensure that provided Private Key password is correct and ' + \
're-import Certificate.'
return False
def run_component_https_cmd(cmd):
retcode, out, err = run_os_command(cmd)
if not retcode == 0:
err = 'Error occured during truststore setup ! :' + out + " : " + err
raise FatalException(1, err)
def run_schema_upgrade(args):
db_title = get_db_type(get_ambari_properties()).title
confirm = get_YN_input(
"Ambari Server configured for %s. Confirm "
"you have made a backup of the Ambari Server database [y/n] (y)? " %
db_title, True)
if not confirm:
print_error_msg("Database backup is not confirmed")
return 1
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg(
"No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
ensure_jdbc_driver_is_installed(args, get_ambari_properties())
print_info_msg('Upgrading database schema', True)
serverClassPath = ServerClassPath(get_ambari_properties(), args)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell(
validate_classpath=True)
set_debug_mode_from_options(args)
debug_mode = get_debug_mode()
debug_start = (debug_mode & 1) or SCHEMA_UPGRADE_DEBUG
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
command = SCHEMA_UPGRADE_HELPER_CMD_DEBUG.format(
jdk_path, class_path,
suspend_mode) if debug_start else SCHEMA_UPGRADE_HELPER_CMD.format(
jdk_path, class_path)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
environ = generate_env(args, ambari_user, current_user)
(retcode, stdout, stderr) = run_os_command(command, env=environ)
upgrade_response = json.loads(stdout)
check_gpl_license_approved(upgrade_response)
print_info_msg(
"Return code from schema upgrade command, retcode = {0}".format(
str(retcode)), True)
if stdout:
print_info_msg("Console output from schema upgrade command:", True)
print_info_msg(stdout, True)
print
if retcode > 0:
print_error_msg(
"Error executing schema upgrade, please check the server logs.")
if stderr:
print_error_msg("Error output from schema upgrade command:")
print_error_msg(stderr)
print
else:
print_info_msg('Schema upgrade completed', True)
return retcode
def run_command(self):
retcode, out, err = run_os_command(self.get_command())
self.returncode = retcode
self.stdoutdata = out
self.stderrdata = err
def setup_truststore(import_cert=False):
if not get_silent():
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
properties = get_ambari_properties()
if get_YN_input("Do you want to configure a truststore [y/n] (y)? ",
True):
#Re-configuration enabled only for option "Setup truststore"
if not import_cert and properties.get_property(SSL_TRUSTSTORE_TYPE_PROPERTY)\
and get_YN_input(
"The truststore is already configured. Do you want to re-configure "
"the truststore [y/n] (y)? ", True):
properties.removeProp(SSL_TRUSTSTORE_TYPE_PROPERTY)
properties.removeProp(SSL_TRUSTSTORE_PATH_PROPERTY)
properties.removeProp(SSL_TRUSTSTORE_PASSWORD_PROPERTY)
truststore_type = get_and_persist_truststore_type(properties)
truststore_path = get_and_persist_truststore_path(properties)
truststore_password = get_and_persist_truststore_password(
properties)
if import_cert:
if get_YN_input(
"Do you want to import a certificate [y/n] (y)? ",
True):
alias = get_validated_string_input(
"Please enter an alias for the certificate: ", "",
None, None, False, False)
run_os_command(
get_delete_cert_command(jdk_path, alias,
truststore_path,
truststore_password))
import_cert_path = get_validated_filepath_input( \
"Enter path to certificate: ", \
"Certificate not found")
run_component_https_cmd(
get_import_cert_command(jdk_path, alias,
truststore_type,
import_cert_path,
truststore_path,
truststore_password))
else:
return
conf_file = find_properties_file()
f = open(conf_file, 'w')
properties.store(f,
"Changed by 'ambari-server setup-security' command")
else:
print "setup-security is not enabled in silent mode."