def extract_views(options):
java_exe_path = get_java_exe_path()
if java_exe_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
vdir = get_value_from_properties(properties, VIEWS_DIR_PROPERTY, configDefaults.DEFAULT_VIEWS_DIR)
files = [f for f in os.listdir(vdir) if os.path.isfile(os.path.join(vdir,f))]
serverClassPath = ServerClassPath(get_ambari_properties(), options)
for f in files:
command = VIEW_EXTRACT_CMD.format(java_exe_path,
serverClassPath.get_full_ambari_classpath_escaped_for_shell(), os.path.join(vdir,f))
retcode, stdout, stderr = run_os_command(command)
if retcode == 0:
sys.stdout.write(f + "\n")
elif retcode == 2:
sys.stdout.write("Error extracting " + f + "\n")
else:
sys.stdout.write(".")
sys.stdout.flush()
print_info_msg("Return code from extraction of view archive " + f + ": " +
str(retcode))
sys.stdout.write("\n")
return 0
def get_default_dbms_name(self):
properties = get_ambari_properties()
default_dbms_name = get_value_from_properties(properties, DEFAULT_DBMS_PROPERTY, "").strip().lower()
if default_dbms_name not in self.DBMS_KEYS_LIST:
return ""
else:
return default_dbms_name
def check_setup_already_done():
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
return properties.get_property(JDK_NAME_PROPERTY) and properties.get_property(JDBC_DATABASE_PROPERTY)
def _cache_jdbc_driver(args):
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
raise FatalException(-1, err)
resources_dir = get_resources_location(properties)
if args.jdbc_driver.endswith(TAR_GZ_ARCHIVE_TYPE):
symlink_name = args.jdbc_db + "-jdbc-driver" + TAR_GZ_ARCHIVE_TYPE
else:
symlink_name = args.jdbc_db + "-jdbc-driver.jar"
jdbc_symlink = os.path.join(resources_dir, symlink_name)
path, jdbc_name = os.path.split(args.jdbc_driver)
if os.path.lexists(jdbc_symlink):
os.remove(jdbc_symlink)
if os.path.isfile(os.path.join(resources_dir, jdbc_name)):
os.remove(os.path.join(resources_dir, jdbc_name))
try:
shutil.copy(args.jdbc_driver, resources_dir)
print "Copying {0} to {1}".format(args.jdbc_driver, resources_dir)
except Exception, e:
err = "Can not copy file {0} to {1} due to: {2} . Please check file " \
"permissions and free disk space.".format(args.jdbc_driver, resources_dir, str(e))
raise FatalException(1, err)
def generate_child_process_param_list(ambari_user, java_exe, class_path,
debug_start, suspend_mode):
from ambari_commons.os_linux import ULIMIT_CMD
properties = get_ambari_properties()
command_base = SERVER_START_CMD_DEBUG if debug_start else SERVER_START_CMD
ulimit_cmd = "%s %s" % (ULIMIT_CMD, str(get_ulimit_open_files(properties)))
command = command_base.format(java_exe,
ambari_provider_module_option,
jvm_args,
class_path,
configDefaults.SERVER_OUT_FILE,
os.path.join(configDefaults.PID_DIR, EXITCODE_NAME),
suspend_mode)
# required to start properly server instance
os.chdir(configDefaults.ROOT_FS_PATH)
#For properly daemonization server should be started using shell as parent
param_list = [locate_file('sh', '/bin'), "-c"]
if is_root() and ambari_user != "root":
# To inherit exported environment variables (especially AMBARI_PASSPHRASE),
# from subprocess, we have to skip --login option of su command. That's why
# we change dir to / (otherwise subprocess can face with 'permission denied'
# errors while trying to list current directory
cmd = "{ulimit_cmd} ; {su} {ambari_user} -s {sh_shell} -c '{command}'".format(ulimit_cmd=ulimit_cmd,
su=locate_file('su', '/bin'), ambari_user=ambari_user,
sh_shell=locate_file('sh', '/bin'), command=command)
else:
cmd = "{ulimit_cmd} ; {command}".format(ulimit_cmd=ulimit_cmd, command=command)
param_list.append(cmd)
return param_list
def setup_jce_policy(args):
if os.path.exists(args[1]):
if not os.path.split(args[1])[0] == configDefaults.SERVER_RESOURCES_DIR:
try:
shutil.copy(args[1], configDefaults.SERVER_RESOURCES_DIR)
except Exception as e:
err = "Fail while trying to copy {0} to {1}. {2}".format(args[1], configDefaults.SERVER_RESOURCES_DIR, e)
raise FatalException(1, err)
else:
err = "Can not run 'setup-jce'. Invalid path {0}.".format(args[1])
raise FatalException(1, err)
properties = get_ambari_properties()
jdk_path = properties.get_property(JAVA_HOME_PROPERTY)
resources_dir = properties.get_property(RESOURCES_DIR_PROPERTY)
zip_name = os.path.split(args[1])[1]
properties.process_pair(JCE_NAME_PROPERTY, zip_name)
print 'Installing JCE policy...'
try:
JDKSetup.unpack_jce_policy(jdk_path, resources_dir, zip_name)
except FatalException as e:
err = 'Installing JCE failed: {0}. Exiting.'.format(e)
raise FatalException(e.code, err)
update_properties(properties)
print 'NOTE: Restart TBDS Server to apply changes' + \
' ("tbds-server restart|stop|start")'
def select_dbms(self, options):
try:
dbms_index = options.database_index
except AttributeError:
db_name = get_value_from_properties(get_ambari_properties(), JDBC_DATABASE_PROPERTY, "").strip().lower()
persistence_type = get_value_from_properties(get_ambari_properties(), PERSISTENCE_TYPE_PROPERTY, "").strip().lower()
if persistence_type == STORAGE_TYPE_LOCAL:
dbms_index = self.DBMS_KEYS_LIST.index("embedded")
elif db_name:
dbms_index = self.DBMS_KEYS_LIST.index(db_name)
else:
dbms_index = self._get_default_dbms_index(options)
if options.must_set_database_options:
n_dbms = 1
dbms_choice_prompt = "==============================================================================\n" \
"Choose one of the following options:\n"
dbms_choices = ''
for desc in self.DBMS_LIST:
if len(desc.storage_name) > 0:
dbms_storage = " ({0})".format(desc.storage_name)
else:
dbms_storage = ""
dbms_choice_prompt += self.DBMS_PROMPT_PATTERN.format(n_dbms, desc.dbms_name, dbms_storage)
dbms_choices += str(n_dbms)
n_dbms += 1
database_num = str(dbms_index + 1)
dbms_choice_prompt += self.DBMS_CHOICE_PROMPT_PATTERN.format(database_num)
dbms_valid_choices = self.JDK_VALID_CHOICES_PATTERN.format(dbms_choices)
database_num = get_validated_string_input(
dbms_choice_prompt,
database_num,
dbms_valid_choices,
"Invalid number.",
False
)
dbms_index = int(database_num) - 1
if dbms_index >= n_dbms:
print_info_msg('Unknown db option, default to {0} {1}.'.format(
self.DBMS_LIST[0].storage_name, self.DBMS_LIST[0].dbms_name))
dbms_index = 0
return dbms_index
def redirect_output_streams(self):
properties = get_ambari_properties()
outFilePath = properties[SERVER_OUT_FILE_KEY]
if (outFilePath is None or outFilePath == ""):
outFilePath = configDefaults.SERVER_OUT_FILE
self._RedirectOutputStreamsToFile(outFilePath)
pass
def run_schema_upgrade(args):
db_title = get_db_type(get_ambari_properties()).title
confirm = get_YN_input("Ambari Server configured for %s. Confirm "
"you have made a backup of the Ambari Server database [y/n] (y)? " % db_title, True)
if not confirm:
print_error_msg("Database backup is not confirmed")
return 1
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
ensure_jdbc_driver_is_installed(args, get_ambari_properties())
print 'Upgrading database schema'
serverClassPath = ServerClassPath(get_ambari_properties(), args)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()
command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, class_path)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
environ = generate_env(args, ambari_user, current_user)
(retcode, stdout, stderr) = run_os_command(command, env=environ)
print_info_msg("Return code from schema upgrade command, retcode = " + str(retcode))
if stdout:
print "Console output from schema upgrade command:"
print stdout
print
if stderr:
print "Error output from schema upgrade command:"
print stderr
print
if retcode > 0:
print_error_msg("Error executing schema upgrade, please check the server logs.")
else:
print_info_msg('Schema upgrade completed')
return retcode
def _reset_database(options):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting tbds properties")
return -1
factory = DBMSConfigFactory()
dbmsAmbari = factory.create(options, properties)
dbmsAmbari.reset_database()
def _cache_jdbc_driver(args):
properties = get_ambari_properties()
if properties == -1:
err = "Error getting tbds properties"
raise FatalException(-1, err)
conf_file = properties.fileName
try:
resources_dir = properties[RESOURCES_DIR_PROPERTY]
except (KeyError), e:
err = 'Property ' + str(e) + ' is not defined at ' + conf_file
raise FatalException(1, err)
def set_current(options):
server_status, pid = is_server_runing()
if not server_status:
err = 'Ambari Server is not running.'
raise FatalException(1, err)
finalize_options = SetCurrentVersionOptions(options)
if finalize_options.no_finalize_options_set():
err = 'Must specify --cluster-name and --version-display-name. Please invoke ambari-server.py --help to print the options.'
raise FatalException(1, err)
admin_login = get_validated_string_input(prompt="Enter Ambari Admin login: "******"Enter Ambari Admin password: "******"Failed to read properties file.")
base_url = get_ambari_server_api_base(properties)
url = base_url + "clusters/{0}/stack_versions".format(finalize_options.cluster_name)
admin_auth = base64.encodestring('%s:%s' % (admin_login, admin_password)).replace('\n', '')
request = urllib2.Request(url)
request.add_header('Authorization', 'Basic %s' % admin_auth)
request.add_header('X-Requested-By', 'ambari')
data = {
"ClusterStackVersions": {
"repository_version": finalize_options.desired_repo_version,
"state": "CURRENT",
"force": "true" if finalize_options.force_repo_version else "false"
}
}
if get_verbose():
sys.stdout.write('\nCalling API ' + url + ' : ' + str(data) + '\n')
request.add_data(json.dumps(data))
request.get_method = lambda: 'PUT'
try:
response = urllib2.urlopen(request)
except urllib2.HTTPError, e:
code = e.getcode()
content = e.read()
err = 'Error during setting current version. Http status code - {0}. \n {1}'.format(
code, content)
raise FatalException(1, err)
def restore_custom_services():
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
try:
resources_dir = properties[RESOURCES_DIR_PROPERTY]
except (KeyError), e:
conf_file = properties.fileName
err = 'Property ' + str(e) + ' is not defined at ' + conf_file
print_error_msg(err)
raise FatalException(1, err)
def check_jdbc_drivers(args):
# create jdbc symlinks if jdbc drivers are available in resources
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
conf_file = properties.fileName
try:
resources_dir = properties[RESOURCES_DIR_PROPERTY]
except (KeyError), e:
err = "Property " + str(e) + " is not defined at " + conf_file
raise FatalException(1, err)
def get_fqdn(timeout=2):
properties = get_ambari_properties()
if properties == -1:
print "Error reading ambari properties"
return None
get_fqdn_service_url = properties[GET_FQDN_SERVICE_URL]
try:
handle = urllib2.urlopen(get_fqdn_service_url, '', timeout)
str = handle.read()
handle.close()
return str
except Exception:
return socket.getfqdn().lower()
def setup_https(args):
if not is_root():
err = 'tbds-server setup-https should be run with ' \
'root-level privileges'
raise FatalException(4, err)
args.exit_message = None
if not get_silent():
properties = get_ambari_properties()
try:
security_server_keys_dir = properties.get_property(SSL_KEY_DIR)
client_api_ssl_port = DEFAULT_SSL_API_PORT if properties.get_property(SSL_API_PORT) in ("") \
else properties.get_property(SSL_API_PORT)
api_ssl = properties.get_property(SSL_API) in ['true']
cert_was_imported = False
cert_must_import = True
if api_ssl:
if get_YN_input("Do you want to disable HTTPS [y/n] (n)? ", False):
properties.process_pair(SSL_API, "false")
cert_must_import=False
else:
properties.process_pair(SSL_API_PORT, \
get_validated_string_input( \
"SSL port ["+str(client_api_ssl_port)+"] ? ", \
str(client_api_ssl_port), \
"^[0-9]{1,5}$", "Invalid port.", False, validatorFunction = is_valid_https_port))
cert_was_imported = import_cert_and_key_action(security_server_keys_dir, properties)
else:
if get_YN_input("Do you want to configure HTTPS [y/n] (y)? ", True):
properties.process_pair(SSL_API_PORT, \
get_validated_string_input("SSL port ["+str(client_api_ssl_port)+"] ? ", \
str(client_api_ssl_port), "^[0-9]{1,5}$", "Invalid port.", False, validatorFunction = is_valid_https_port))
cert_was_imported = import_cert_and_key_action(security_server_keys_dir, properties)
else:
return False
if cert_must_import and not cert_was_imported:
print 'Setup of HTTPS failed. Exiting.'
return False
conf_file = find_properties_file()
f = open(conf_file, 'w')
properties.store(f, "Changed by 'tbds-server setup-https' command")
ambari_user = read_ambari_user()
if ambari_user:
adjust_directory_permissions(ambari_user)
return True
except (KeyError), e:
err = 'Property ' + str(e) + ' is not defined'
raise FatalException(1, err)
def _is_jdbc_user_changed(database_username):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return None
previos_user = get_value_from_properties(properties, JDBC_USER_NAME_PROPERTY, "")
if previos_user and database_username:
if previos_user != database_username:
return True
else:
return False
return None
def _setup_database(options):
properties = get_ambari_properties()
if properties == -1:
raise FatalException(-1, "Error getting ambari properties")
factory = DBMSConfigFactory()
dbmsAmbari = factory.create(options, properties, "Ambari")
resultA = dbmsAmbari.configure_database(properties)
# Now save the properties file
if resultA:
update_properties(properties)
dbmsAmbari.setup_database()
def download_and_install_jdk(options):
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
raise FatalException(-1, err)
jdkSetup = JDKSetup()
jdkSetup.download_and_install_jdk(options, properties)
if jdkSetup.jdk_index != jdkSetup.custom_jdk_number:
jdkSetup.download_and_unpack_jce_policy(properties)
update_properties(properties)
return 0
def move_user_custom_actions():
print_info_msg('Moving *.py files from custom_actions to custom_actions/scripts')
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
try:
resources_dir = properties[RESOURCES_DIR_PROPERTY]
except (KeyError), e:
conf_file = properties.fileName
err = 'Property ' + str(e) + ' is not defined at ' + conf_file
print_error_msg(err)
raise FatalException(1, err)
def verify_setup_allowed():
if get_silent():
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting tbds properties")
return -1
isSecure = get_is_secure(properties)
if isSecure:
(isPersisted, masterKeyFile) = get_is_persisted(properties)
if not isPersisted:
print "ERROR: Cannot run silent 'setup' with password encryption enabled " \
"and Master Key not persisted."
print "TBDS Server 'setup' exiting."
return 1
return 0
def _reset_database(options):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
persistence_type = properties[PERSISTENCE_TYPE_PROPERTY]
if persistence_type == "remote":
err = 'Ambari doesn\'t support resetting exernal DB automatically. ' \
'To reset Ambari Server schema you must first drop and then create it ' \
'using DDL scripts from "/var/lib/ambari-server/resources/"'
raise FatalException(1, err)
else:
factory = DBMSConfigFactory()
dbmsAmbari = factory.create(options, properties)
dbmsAmbari.reset_database()
def _setup_remote_database(self):
properties = get_ambari_properties()
if properties == -1:
err = 'Error getting ambari properties'
print_error_msg(err)
raise FatalException(-1, err)
if self.ensure_jdbc_driver_installed(properties):
print 'Configuring remote database connection properties...'
retcode = self._setup_remote_db()
if retcode == -1:
err = "Remote database setup aborted."
raise NonFatalException(err)
if not retcode == 0:
err = 'Error while configuring connection properties. Exiting'
raise FatalException(retcode, err)
def setup_component_https(component, command, property, alias):
if not get_silent():
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"tbds-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
properties = get_ambari_properties()
use_https = properties.get_property(property) in ['true']
if use_https:
if get_YN_input("Do you want to disable HTTPS for " + component + " [y/n] (n)? ", False):
truststore_path = get_truststore_path(properties)
truststore_password = get_truststore_password(properties)
run_component_https_cmd(get_delete_cert_command(jdk_path, alias, truststore_path, truststore_password))
properties.process_pair(property, "false")
else:
return
else:
if get_YN_input("Do you want to configure HTTPS for " + component + " [y/n] (y)? ", True):
truststore_type = get_truststore_type(properties)
truststore_path = get_truststore_path(properties)
truststore_password = get_truststore_password(properties)
run_os_command(get_delete_cert_command(jdk_path, alias, truststore_path, truststore_password))
import_cert_path = get_validated_filepath_input( \
"Enter path to " + component + " Certificate: ", \
"Certificate not found")
run_component_https_cmd(get_import_cert_command(jdk_path, alias, truststore_type, import_cert_path, truststore_path, truststore_password))
properties.process_pair(property, "true")
else:
return
conf_file = find_properties_file()
f = open(conf_file, 'w')
properties.store(f, "Changed by 'tbds-server " + command + "' command")
else:
print command + " is not enabled in silent mode."
def generate_env(ambari_user, current_user):
properties = get_ambari_properties()
isSecure = get_is_secure(properties)
(isPersisted, masterKeyFile) = get_is_persisted(properties)
environ = os.environ.copy()
# Need to handle master key not persisted scenario
if isSecure and not masterKeyFile:
prompt = False
masterKey = environ.get(SECURITY_KEY_ENV_VAR_NAME)
if masterKey is not None and masterKey != "":
pass
else:
keyLocation = environ.get(SECURITY_MASTER_KEY_LOCATION)
if keyLocation is not None:
try:
# Verify master key can be read by the java process
with open(keyLocation, 'r'):
pass
except IOError:
print_warning_msg("Cannot read Master key from path specified in "
"environemnt.")
prompt = True
else:
# Key not provided in the environment
prompt = True
if prompt:
import pwd
masterKey = get_original_master_key(properties)
tempDir = tempfile.gettempdir()
tempFilePath = tempDir + os.sep + "masterkey"
save_master_key(masterKey, tempFilePath, True)
if ambari_user != current_user:
uid = pwd.getpwnam(ambari_user).pw_uid
gid = pwd.getpwnam(ambari_user).pw_gid
os.chown(tempFilePath, uid, gid)
else:
os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE)
if tempFilePath is not None:
environ[SECURITY_MASTER_KEY_LOCATION] = tempFilePath
return environ
def adjust_directory_permissions(ambari_user):
properties = get_ambari_properties()
bootstrap_dir = os.path.abspath(get_value_from_properties(properties, BOOTSTRAP_DIR_PROPERTY))
print_info_msg("Cleaning bootstrap directory ({0}) contents...".format(bootstrap_dir))
shutil.rmtree(bootstrap_dir, True) #Ignore the non-existent dir error
#Protect against directories lingering around
del_attempts = 0
while os.path.exists(bootstrap_dir) and del_attempts < 100:
time.sleep(50)
del_attempts += 1
if not os.path.exists(bootstrap_dir):
try:
os.makedirs(bootstrap_dir)
except Exception, ex:
print_warning_msg("Failed recreating the bootstrap directory: {0}".format(str(ex)))
pass
def run_metainfo_upgrade(args, keyValueMap=None):
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
retcode = 1
if keyValueMap:
serverClassPath = ServerClassPath(get_ambari_properties(), args)
command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, serverClassPath.get_full_ambari_classpath_escaped_for_shell(),
'updateMetaInfo',
"'" + json.dumps(keyValueMap) + "'")
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode))
if retcode > 0:
print_error_msg("Error executing metainfo upgrade, please check the "
"server logs.")
return retcode
def configure_os_settings():
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
try:
conf_os_type = properties[OS_TYPE_PROPERTY]
if conf_os_type != '':
print_info_msg("os_type already set in the properties file")
return 0
except (KeyError):
print_error_msg("os_type is not set in the properties file. Setting it now.")
# to check server/agent compatibility
master_os_family = OS_FAMILY + OS_VERSION
# to check supported os_types
master_os_type = OS_TYPE + OS_VERSION
write_property(OS_FAMILY_PROPERTY, master_os_family)
write_property(OS_TYPE_PROPERTY, master_os_type)
return 0
def upgrade_stack(args):
if not is_root():
err = 'Ambari-server upgradestack should be run with ' \
'root-level privileges'
raise FatalException(4, err)
check_database_name_property()
try:
stack_id = args[1]
except IndexError:
#stack_id is mandatory
raise FatalException("Invalid number of stack upgrade arguments")
try:
repo_url = args[2]
except IndexError:
repo_url = None
try:
repo_url_os = args[3]
except IndexError:
repo_url_os = None
parser = optparse.OptionParser()
parser.add_option("-d", type="int", dest="database_index")
db = get_ambari_properties()[JDBC_DATABASE_PROPERTY]
idx = LINUX_DBMS_KEYS_LIST.index(db)
(options, opt_args) = parser.parse_args(["-d {0}".format(idx)])
stack_name, stack_version = stack_id.split(STACK_NAME_VER_SEP)
retcode = run_stack_upgrade(options, stack_name, stack_version, repo_url, repo_url_os)
if not retcode == 0:
raise FatalException(retcode, 'Stack upgrade failed.')
return retcode
def upgrade_local_repo(args):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
stack_location = get_stack_location(properties)
stack_root_local = os.path.join(stack_location, "HDPLocal")
if not os.path.exists(stack_root_local):
print_info_msg("HDPLocal stack directory does not exist, skipping")
return
stack_root = os.path.join(stack_location, "HDP")
if not os.path.exists(stack_root):
print_info_msg("HDP stack directory does not exist, skipping")
return
for stack_version_local in os.listdir(stack_root_local):
repo_file_local = os.path.join(stack_root_local, stack_version_local, "repos", "repoinfo.xml.rpmsave")
if not os.path.exists(repo_file_local):
repo_file_local = os.path.join(stack_root_local, stack_version_local, "repos", "repoinfo.xml")
repo_file = os.path.join(stack_root, stack_version_local, "repos", "repoinfo.xml")
print_info_msg("Local repo file: " + repo_file_local)
print_info_msg("Repo file: " + repo_file_local)
metainfo_update_items = {}
if os.path.exists(repo_file_local) and os.path.exists(repo_file):
local_values = load_stack_values(stack_version_local, repo_file_local)
repo_values = load_stack_values(stack_version_local, repo_file)
for k, v in local_values.iteritems():
if repo_values.has_key(k):
local_url = local_values[k]
repo_url = repo_values[k]
if repo_url != local_url:
metainfo_update_items[k] = local_url
run_metainfo_upgrade(args, metainfo_update_items)
def setup(options):
logger.info("Setup ambari-server.")
if options.only_silent:
if check_setup_already_done():
print "Nothing was done. Ambari Setup already performed and cannot re-run setup in silent mode. Use \"ambari-server setup\" command without -s option to change Ambari setup."
sys.exit(0)
retcode = verify_setup_allowed(options)
if not retcode == 0:
raise FatalException(1, None)
if not is_root():
warn_msg = configDefaults.MESSAGE_WARN_SETUP_NOT_ROOT
print warn_msg
# proceed jdbc properties if they were set
if _check_jdbc_options(options):
proceedJDBCProperties(options)
return
(retcode, err) = disable_security_enhancements()
if not retcode == 0:
raise FatalException(retcode, err)
#Create ambari user, if needed
(retcode, register_service, svc_user, svc_password) = check_ambari_user(options)
if not retcode == 0:
err = 'Failed to create user. Exiting.'
raise FatalException(retcode, err)
print configDefaults.MESSAGE_CHECK_FIREWALL
check_firewall()
# proceed jdbc properties if they were set
if _check_jdbc_options(options):
proceedJDBCProperties(options)
print 'Checking JDK...'
try:
download_and_install_jdk(options)
except FatalException as e:
err = 'Downloading or installing JDK failed: {0}. Exiting.'.format(e)
raise FatalException(e.code, err)
print 'Checking GPL software agreement...'
write_gpl_license_accepted(default_prompt_value=options.accept_gpl)
print 'Completing setup...'
retcode = configure_os_settings()
if not retcode == 0:
err = 'Configure of OS settings in ambari.properties failed. Exiting.'
raise FatalException(retcode, err)
print 'Configuring database...'
prompt_db_properties(options)
#DB setup should be done last after doing any setup.
_setup_database(options)
check_jdbc_drivers(options)
if not options.skip_view_extraction:
print 'Extracting system views...'
retcode = extract_views(options)
if not retcode == 0:
err = 'Error while extracting system views. Exiting'
raise FatalException(retcode, err)
json_url = get_json_url_from_repo_file()
if json_url:
print "Ambari repo file contains latest json url {0}, updating stacks repoinfos with it...".format(json_url)
properties = get_ambari_properties()
stack_root = get_stack_location(properties)
update_latest_in_repoinfos_for_all_stacks(stack_root, json_url)
else:
print "Ambari repo file doesn't contain latest json url, skipping repoinfos modification"
# we've already done this, but new files were created so run it one time.
adjust_directory_permissions(svc_user)
service_setup(register_service, svc_user, svc_password)
def server_process_main(options, scmStatus=None):
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
raise FatalException(-1, err)
properties_for_print = []
logger.info("Ambari server properties config:")
for key, value in properties.getPropertyDict().items():
if "passwd" not in key and "password" not in key:
properties_for_print.append(key + "=" + value)
logger.info(properties_for_print)
# debug mode, including stop Java process at startup
try:
set_debug_mode_from_options(options)
except AttributeError:
pass
if not check_reverse_lookup():
print_warning_msg(
"The hostname was not found in the reverse DNS lookup. "
"This may result in incorrect behavior. "
"Please check the DNS setup and fix the issue.")
check_database_name_property()
parse_properties_file(options)
is_active_instance = get_is_active_instance()
if not is_active_instance:
print_warning_msg(
"This instance of ambari server is not designated as active. Cannot start ambari server."
)
err = "This is not an active instance. Shutting down..."
raise FatalException(1, err)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
print_info_msg("Ambari Server is not running...")
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
if not options.skip_properties_validation:
missing_properties = get_missing_properties(properties)
if missing_properties:
err = "Required properties are not found: " + str(missing_properties) + ". To skip properties validation " \
"use \"--skip-properties-validation\""
raise FatalException(1, err)
# Preparations
if is_root():
print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT
ensure_jdbc_driver_is_installed(options, properties)
ensure_dbms_is_running(options, properties, scmStatus)
if scmStatus is not None:
scmStatus.reportStartPending()
refresh_stack_hash(properties)
if scmStatus is not None:
scmStatus.reportStartPending()
ensure_server_security_is_configured()
if scmStatus is not None:
scmStatus.reportStartPending()
java_exe = get_java_exe_path()
serverClassPath = ServerClassPath(properties, options)
debug_mode = get_debug_mode()
debug_start = (debug_mode & 1) or SERVER_START_DEBUG
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
environ = generate_env(options, ambari_user, current_user)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell(
validate_classpath=True)
if options.skip_database_check:
global jvm_args
jvm_args += " -DskipDatabaseConsistencyCheck"
print "Ambari Server is starting with the database consistency check skipped. Do not make any changes to your cluster " \
"topology or perform a cluster upgrade until you correct the database consistency issues. See \"" \
+ configDefaults.DB_CHECK_LOG + "\" for more details on the consistency issues."
properties.process_pair(CHECK_DATABASE_SKIPPED_PROPERTY, "true")
else:
print "Ambari database consistency check started..."
properties.process_pair(CHECK_DATABASE_SKIPPED_PROPERTY, "false")
command = CHECK_DATABASE_HELPER_CMD.format(java_exe, class_path)
(retcode, stdout, stderr) = run_os_command(command, env=environ)
if retcode > 0:
print str(stdout)
raise FatalException(
1, 'Database check failed to complete. Please check ' +
configDefaults.SERVER_LOG_FILE + ' and ' +
configDefaults.DB_CHECK_LOG + ' for more information.')
else:
print str(stdout)
print "Ambari database consistency check finished"
if not stdout.startswith("No errors"):
sys.exit(1)
update_properties(properties)
param_list = generate_child_process_param_list(ambari_user, java_exe,
class_path, debug_start,
suspend_mode)
# The launched shell process and sub-processes should have a group id that
# is different from the parent.
def make_process_independent():
if IS_FOREGROUND: # upstart script is not able to track process from different pgid.
return
processId = os.getpid()
if processId > 0:
try:
os.setpgid(processId, processId)
except OSError, e:
print_warning_msg('setpgid({0}, {0}) failed - {1}'.format(
pidJava, str(e)))
pass
def setup_https(args):
if not is_root():
warn = 'ambari-server setup-https is run as ' \
'non-root user, some sudo privileges might be required'
print warn
args.exit_message = None
if not get_silent():
properties = get_ambari_properties()
try:
security_server_keys_dir = properties.get_property(SSL_KEY_DIR)
client_api_ssl_port = DEFAULT_SSL_API_PORT if properties.get_property(SSL_API_PORT) in ("") \
else properties.get_property(SSL_API_PORT)
api_ssl = properties.get_property(SSL_API) in ['true']
client_api_ssl_port_old_value = properties.get_property(
SSL_API_PORT)
api_ssl_old_value = properties.get_property(SSL_API)
cert_was_imported = False
cert_must_import = True
if api_ssl:
if get_YN_input("Do you want to disable HTTPS [y/n] (n)? ",
False):
properties.process_pair(SSL_API, "false")
cert_must_import = False
else:
properties.process_pair(SSL_API_PORT, \
get_validated_string_input( \
"SSL port ["+str(client_api_ssl_port)+"] ? ", \
str(client_api_ssl_port), \
"^[0-9]{1,5}$", "Invalid port.", False, validatorFunction = is_valid_https_port))
cert_was_imported = import_cert_and_key_action(
security_server_keys_dir, properties)
else:
if get_YN_input("Do you want to configure HTTPS [y/n] (y)? ",
True):
properties.process_pair(SSL_API_PORT, \
get_validated_string_input("SSL port ["+str(client_api_ssl_port)+"] ? ", \
str(client_api_ssl_port), "^[0-9]{1,5}$", "Invalid port.", False, validatorFunction = is_valid_https_port))
cert_was_imported = import_cert_and_key_action(
security_server_keys_dir, properties)
else:
return False
if cert_must_import and not cert_was_imported:
print 'Setup of HTTPS failed. Exiting.'
return False
conf_file = find_properties_file()
f = open(conf_file, 'w')
properties.store(f,
"Changed by 'ambari-server setup-https' command")
if api_ssl_old_value != properties.get_property(SSL_API) \
or client_api_ssl_port_old_value != properties.get_property(SSL_API_PORT):
print "Ambari server URL changed. To make use of the Tez View in Ambari " \
"please update the property tez.tez-ui.history-url.base in tez-site"
ambari_user = read_ambari_user()
if ambari_user:
adjust_directory_permissions(ambari_user)
return True
except (KeyError), e:
err = 'Property ' + str(e) + ' is not defined'
raise FatalException(1, err)
def setup_sso(options):
print_info_msg("Setup SSO.")
server_status, pid = is_server_runing()
if not server_status:
err = 'Ambari Server is not running.'
raise FatalException(1, err)
if not get_silent():
validate_options(options)
ambari_properties = get_ambari_properties()
admin_login, admin_password = get_ambari_admin_username_password_pair(
options)
properties = get_sso_properties(ambari_properties, admin_login,
admin_password)
if not options.sso_enabled:
ambari_auth_enabled = get_value_from_dictionary(
properties, AMBARI_SSO_AUTH_ENABLED)
manage_services = get_value_from_dictionary(
properties, SSO_MANAGE_SERVICES)
if ambari_auth_enabled or manage_services:
if (ambari_auth_enabled and 'true' == ambari_auth_enabled) or \
(manage_services and 'true' == manage_services):
sso_status = "enabled"
else:
sso_status = "disabled"
else:
sso_status = "not configured"
sys.stdout.write("\nSSO is currently %s\n" % sso_status)
if sso_status == "enabled":
enable_sso = not get_YN_input(
"Do you want to disable SSO authentication [y/n] (n)? ",
False)
elif get_YN_input(
"Do you want to configure SSO authentication [y/n] (y)? ",
True):
enable_sso = True
else:
return False
else:
enable_sso = options.sso_enabled == 'true'
if enable_sso:
populate_sso_provider_url(options, properties)
populate_sso_public_cert(options, properties)
populate_ambari_requires_sso(options, properties)
populate_service_management(options, properties, ambari_properties,
admin_login, admin_password)
populate_jwt_cookie_name(options, properties)
populate_jwt_audiences(options, properties)
update_sso_conf(ambari_properties, properties, admin_login,
admin_password)
else:
remove_sso_conf(ambari_properties, admin_login, admin_password)
else:
warning = "setup-sso is not enabled in silent mode."
raise NonFatalException(warning)
pass
def setup_trusted_proxy(options):
print_info_msg("Setup Trusted Proxy")
server_status, pid = is_server_runing()
if not server_status:
err = 'Ambari Server is not running.'
raise FatalException(1, err)
if not get_silent():
validate_options(options)
ambari_properties = get_ambari_properties()
admin_login, admin_password = get_ambari_admin_username_password_pair(
options)
properties = get_trusted_proxy_properties(ambari_properties,
admin_login, admin_password)
if not options.tproxy_enabled:
tproxy_support_enabled = get_value_from_dictionary(
properties, TPROXY_SUPPORT_ENABLED)
if tproxy_support_enabled:
if 'true' == tproxy_support_enabled:
tproxy_status = "enabled"
else:
tproxy_status = "disabled"
else:
tproxy_status = "not configured"
print_info_msg("\nTrusted Proxy support is currently %s\n" %
tproxy_status)
if tproxy_status == "enabled":
enable_tproxy = not get_YN_input(
"Do you want to disable Trusted Proxy support [y/n] (n)? ",
False)
elif get_YN_input(
"Do you want to configure Trusted Proxy Support [y/n] (y)? ",
True):
enable_tproxy = True
else:
return False
else:
enable_tproxy = options.tproxy_enabled == 'true'
if enable_tproxy:
properties[TPROXY_SUPPORT_ENABLED] = "true"
if not options.tproxy_configuration_file_path:
add_new_trusted_proxy = add_new_trusted_proxy_config(
properties)
while add_new_trusted_proxy:
add_new_trusted_proxy = add_new_trusted_proxy_config(
properties)
else:
parse_trusted_configuration_file(
options.tproxy_configuration_file_path, properties)
update_tproxy_conf(ambari_properties, properties, admin_login,
admin_password)
else:
remove_tproxy_conf(ambari_properties, admin_login, admin_password)
else:
warning = "setup-trusted-proxy is not enabled in silent mode."
raise NonFatalException(warning)
pass
def setup_master_key(options):
if not is_root():
err = 'Ambari-server setup should be run with ' \
'root-level privileges'
raise FatalException(4, err)
properties = get_ambari_properties()
if properties == -1:
raise FatalException(1, "Failed to read properties file.")
db_windows_auth_prop = properties.get_property(JDBC_USE_INTEGRATED_AUTH_PROPERTY)
db_sql_auth = False if db_windows_auth_prop and db_windows_auth_prop.lower() == 'true' else True
db_password = properties.get_property(JDBC_PASSWORD_PROPERTY)
# Encrypt passwords cannot be called before setup
if db_sql_auth and not db_password:
print 'Please call "setup" before "encrypt-passwords". Exiting...'
return 1
# Check configuration for location of master key
isSecure = get_is_secure(properties)
(isPersisted, masterKeyFile) = get_is_persisted(properties)
# Read clear text DB password from file
if db_sql_auth and not is_alias_string(db_password) and os.path.isfile(db_password):
with open(db_password, 'r') as passwdfile:
db_password = passwdfile.read()
ldap_password = properties.get_property(LDAP_MGR_PASSWORD_PROPERTY)
if ldap_password:
# Read clear text LDAP password from file
if not is_alias_string(ldap_password) and os.path.isfile(ldap_password):
with open(ldap_password, 'r') as passwdfile:
ldap_password = passwdfile.read()
ts_password = properties.get_property(SSL_TRUSTSTORE_PASSWORD_PROPERTY)
resetKey = False
masterKey = None
if isSecure:
print "Password encryption is enabled."
resetKey = get_YN_input("Do you want to reset Master Key? [y/n] (n): ", False)
# For encrypting of only unencrypted passwords without resetting the key ask
# for master key if not persisted.
if isSecure and not isPersisted and not resetKey:
print "Master Key not persisted."
masterKey = get_original_master_key(properties)
pass
# Make sure both passwords are clear-text if master key is lost
if resetKey:
if not isPersisted:
print "Master Key not persisted."
masterKey = get_original_master_key(properties)
# Unable get the right master key or skipped question <enter>
if not masterKey:
print "To disable encryption, do the following:"
print "- Edit " + find_properties_file() + \
" and set " + SECURITY_IS_ENCRYPTION_ENABLED + " = " + "false."
err = "{0} is already encrypted. Please call {1} to store unencrypted" \
" password and call 'encrypt-passwords' again."
if db_sql_auth and db_password and is_alias_string(db_password):
print err.format('- Database password', "'" + SETUP_ACTION + "'")
if ldap_password and is_alias_string(ldap_password):
print err.format('- LDAP manager password', "'" + LDAP_SETUP_ACTION + "'")
if ts_password and is_alias_string(ts_password):
print err.format('TrustStore password', "'" + LDAP_SETUP_ACTION + "'")
return 1
pass
pass
pass
# Read back any encrypted passwords
if db_sql_auth and db_password and is_alias_string(db_password):
db_password = read_passwd_for_alias(JDBC_RCA_PASSWORD_ALIAS, masterKey)
if ldap_password and is_alias_string(ldap_password):
ldap_password = read_passwd_for_alias(LDAP_MGR_PASSWORD_ALIAS, masterKey)
if ts_password and is_alias_string(ts_password):
ts_password = read_passwd_for_alias(SSL_TRUSTSTORE_PASSWORD_ALIAS, masterKey)
# Read master key, if non-secure or reset is true
if resetKey or not isSecure:
masterKey = read_master_key(resetKey)
persist = get_YN_input("Do you want to persist master key. If you choose " \
"not to persist, you need to provide the Master " \
"Key while starting the ambari server as an env " \
"variable named " + SECURITY_KEY_ENV_VAR_NAME + \
" or the start will prompt for the master key."
" Persist [y/n] (y)? ", True)
if persist:
save_master_key(options, masterKey, get_master_key_location(properties) + os.sep +
SECURITY_MASTER_KEY_FILENAME, persist)
elif not persist and masterKeyFile:
try:
os.remove(masterKeyFile)
print_info_msg("Deleting master key file at location: " + str(
masterKeyFile))
except Exception, e:
print 'ERROR: Could not remove master key file. %s' % e
# Blow up the credential store made with previous key, if any
store_file = get_credential_store_location(properties)
if os.path.exists(store_file):
try:
os.remove(store_file)
except:
print_warning_msg("Failed to remove credential store file.")
pass
pass
def getServiceConfigurationRecommendations(self, configurations,
clusterData, services, hosts):
nifi = self.getServicesSiteProperties(services, "nifi")
if "ranger-env" in services["configurations"] and "ranger-nifi-plugin-properties" in services["configurations"] and \
"ranger-nifi-plugin-enabled" in services["configurations"]["ranger-env"]["properties"]:
putNiFiRangerPluginProperty = self.putProperty(
configurations, "ranger-nifi-plugin-properties", services)
rangerEnvNiFiPluginProperty = services["configurations"][
"ranger-env"]["properties"]["ranger-nifi-plugin-enabled"]
putNiFiRangerPluginProperty("ranger-nifi-plugin-enabled",
rangerEnvNiFiPluginProperty)
if rangerEnvNiFiPluginProperty == 'Yes' and \
"nifi.authentication" in services["configurations"]["ranger-nifi-plugin-properties"]["properties"] and \
"nifi.node.ssl.isenabled" in services["configurations"]["nifi-ambari-ssl-config"]["properties"]:
nifiAmbariSSLConfig = 'SSL' if services["configurations"][
"nifi-ambari-ssl-config"]["properties"][
"nifi.node.ssl.isenabled"] == 'true' else 'NONE'
putNiFiRangerPluginProperty("nifi.authentication",
nifiAmbariSSLConfig)
# Recommend Ranger supported service's audit properties
ranger_audit_dict = [{
'filename':
'ranger-env',
'configname':
'xasecure.audit.destination.solr',
'target_configname':
'xasecure.audit.destination.solr'
}, {
'filename':
'ranger-env',
'configname':
'xasecure.audit.destination.hdfs',
'target_configname':
'xasecure.audit.destination.hdfs'
}, {
'filename':
'ranger-env',
'configname':
'xasecure.audit.destination.hdfs.dir',
'target_configname':
'xasecure.audit.destination.hdfs.dir'
}, {
'filename':
'ranger-admin-site',
'configname':
'ranger.audit.solr.urls',
'target_configname':
'xasecure.audit.destination.solr.urls'
}, {
'filename':
'ranger-admin-site',
'configname':
'ranger.audit.solr.zookeepers',
'target_configname':
'xasecure.audit.destination.solr.zookeepers'
}]
for item in ranger_audit_dict:
if item['filename'] in services['configurations'] and item[
'configname'] in services['configurations'][
item['filename']]['properties']:
if item['filename'] in configurations and item[
'configname'] in configurations[
item['filename']]['properties']:
rangerAuditProperty = configurations[
item['filename']]['properties'][item['configname']]
else:
rangerAuditProperty = services['configurations'][
item['filename']]['properties'][item['configname']]
putNifiRangerAuditProperty = self.putProperty(
configurations, "ranger-nifi-audit", services)
putNifiRangerAuditProperty(item['target_configname'],
rangerAuditProperty)
servicesList = [
service['StackServices']['service_name']
for service in services['services']
]
if "RANGER" in servicesList and "ranger-admin-site" in services[
'configurations']:
nifi_user = services['configurations']['nifi-env']['properties'][
'nifi_user']
putRangerAdminSiteProperty = self.putProperty(
configurations, "ranger-admin-site", services)
putRangerAdminSiteProperty("ranger.plugins.nifi.serviceuser",
nifi_user)
properties = get_ambari_properties()
ambari_version = get_ambari_version(properties)
if not (ambari_version) or not (ambari_version.startswith('2.5')):
putNiFiLogSearchConfAttribute = self.putPropertyAttribute(
configurations, "nifi-logsearch-conf")
putNiFiLogSearchConfAttribute('service_name', 'visible', 'false')
putNiFiLogSearchConfAttribute('component_mappings', 'visible',
'false')
putNiFiLogSearchConfAttribute('content', 'visible', 'false')
def run_db_cleanup(options):
if validate_args(options):
return 1
db_title = get_db_type(get_ambari_properties()).title
confirmBackup = get_YN_input(
"Ambari Server configured for {0}. Confirm you have made a backup of the Ambari Server database [y/n]"
.format(db_title), True)
if not confirmBackup:
print_info_msg("Ambari Server Database cleanup aborted")
return 0
status, stateDesc = is_server_runing()
if status:
print_error_msg(
"The database cleanup cannot proceed while Ambari Server is running. Please shut down Ambari first."
)
return 1
confirm = get_YN_input(
"Ambari server is using db type {0}. Cleanable database entries older than {1} will be cleaned up. Proceed [y/n]"
.format(db_title, options.cleanup_from_date), True)
if not confirm:
print_info_msg("Ambari Server Database cleanup aborted")
return 0
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg(
"No JDK found, please run the \"setup\" command to install a JDK automatically or install any "
"JDK manually to {0}".format(configDefaults.JDK_INSTALL_DIR))
return 1
ensure_jdbc_driver_is_installed(options, get_ambari_properties())
serverClassPath = ServerClassPath(get_ambari_properties(), options)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
environ = generate_env(options, ambari_user, current_user)
print "Cleaning up the database ..."
command = DB_CLEANUP_CMD.format(jdk_path, class_path, options.cluster_name,
options.cleanup_from_date)
(retcode, stdout, stderr) = run_os_command(command, env=environ)
print_info_msg("Return code from database cleanup command, retcode = " +
str(retcode))
if stdout:
print "Console output from database cleanup command:"
print stdout
print
if stderr:
print "Error output from database cleanup command:"
print stderr
print
if retcode > 0:
print_error_msg(
"Error wncountered while cleaning up the Ambari Server Database. Check the ambari-server.log for details."
)
else:
print "Cleanup completed. Check the ambari-server.log for details."
return retcode
def sync_ldap(options):
if not is_root():
err = 'Ambari-server sync-ldap should be run with ' \
'root-level privileges'
raise FatalException(4, err)
server_status, pid = is_server_runing()
if not server_status:
err = 'Ambari Server is not running.'
raise FatalException(1, err)
properties = get_ambari_properties()
if properties == -1:
raise FatalException(1, "Failed to read properties file.")
ldap_configured = properties.get_property(IS_LDAP_CONFIGURED)
if ldap_configured != 'true':
err = "LDAP is not configured. Run 'ambari-server setup-ldap' first."
raise FatalException(1, err)
# set ldap sync options
ldap_sync_options = LdapSyncOptions(options)
if ldap_sync_options.no_ldap_sync_options_set():
err = 'Must specify a sync option (all, existing, users or groups). Please invoke ambari-server.py --help to print the options.'
raise FatalException(1, err)
admin_login = get_validated_string_input(prompt="Enter Ambari Admin login: "******"Enter Ambari Admin password: "******"Event":{"specs":[{"principal_type":"users","sync_type":"all"},{"principal_type":"groups","sync_type":"all"}]}}]
elif ldap_sync_options.ldap_sync_existing:
sys.stdout.write('Syncing existing.')
bodies = [{"Event":{"specs":[{"principal_type":"users","sync_type":"existing"},{"principal_type":"groups","sync_type":"existing"}]}}]
else:
sys.stdout.write('Syncing specified users and groups.')
bodies = [{"Event":{"specs":[]}}]
body = bodies[0]
events = body['Event']
specs = events['specs']
if ldap_sync_options.ldap_sync_users is not None:
new_specs = [{"principal_type":"users","sync_type":"specific","names":""}]
get_ldap_event_spec_names(ldap_sync_options.ldap_sync_users, specs, new_specs)
if ldap_sync_options.ldap_sync_groups is not None:
new_specs = [{"principal_type":"groups","sync_type":"specific","names":""}]
get_ldap_event_spec_names(ldap_sync_options.ldap_sync_groups, specs, new_specs)
if get_verbose():
sys.stdout.write('\nCalling API ' + url + ' : ' + str(bodies) + '\n')
request.add_data(json.dumps(bodies))
request.get_method = lambda: 'POST'
try:
response = urllib2.urlopen(request)
except Exception as e:
err = 'Sync event creation failed. Error details: %s' % e
raise FatalException(1, err)
response_status_code = response.getcode()
if response_status_code != 201:
err = 'Error during syncing. Http status code - ' + str(response_status_code)
raise FatalException(1, err)
response_body = json.loads(response.read())
url = response_body['resources'][0]['href']
request = urllib2.Request(url)
request.add_header('Authorization', 'Basic %s' % admin_auth)
request.add_header('X-Requested-By', 'ambari')
body = [{"LDAP":{"synced_groups":"*","synced_users":"*"}}]
request.add_data(json.dumps(body))
request.get_method = lambda: 'GET'
request_in_progress = True
while request_in_progress:
sys.stdout.write('.')
sys.stdout.flush()
try:
response = urllib2.urlopen(request)
except Exception as e:
request_in_progress = False
err = 'Sync event check failed. Error details: %s' % e
raise FatalException(1, err)
response_status_code = response.getcode()
if response_status_code != 200:
err = 'Error during syncing. Http status code - ' + str(response_status_code)
raise FatalException(1, err)
response_body = json.loads(response.read())
sync_info = response_body['Event']
if sync_info['status'] == 'ERROR':
raise FatalException(1, str(sync_info['status_detail']))
elif sync_info['status'] == 'COMPLETE':
print '\n\nCompleted LDAP Sync.'
print 'Summary:'
for principal_type, summary in sync_info['summary'].iteritems():
print ' {0}:'.format(principal_type)
for action, amount in summary.iteritems():
print ' {0} = {1!s}'.format(action, amount)
request_in_progress = False
else:
time.sleep(1)
sys.stdout.write('\n')
sys.stdout.flush()
def upgrade(args):
print_info_msg("Upgrade Ambari Server", True)
if not is_root():
err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT
raise FatalException(4, err)
print_info_msg('Updating Ambari Server properties in {0} ...'.format(AMBARI_PROPERTIES_FILE), True)
retcode = update_ambari_properties()
if not retcode == 0:
err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
print_info_msg('Updating Ambari Server properties in {0} ...'.format(AMBARI_ENV_FILE), True)
retcode = update_ambari_env()
if not retcode == 0:
err = AMBARI_ENV_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
retcode = update_krb_jaas_login_properties()
if retcode == -2:
pass # no changes done, let's be silent
elif retcode == 0:
print_info_msg("File {0} updated.".format(AMBARI_KRB_JAAS_LOGIN_FILE), True)
elif not retcode == 0:
err = AMBARI_KRB_JAAS_LOGIN_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
restore_custom_services()
replay_mpack_logs()
try:
update_database_name_property(upgrade=True)
except FatalException:
return -1
# Ignore the server version & database options passed via command-line arguments
parse_properties_file(args)
#TODO check database version
change_objects_owner(args)
retcode = run_schema_upgrade(args)
if not retcode == 0:
print_error_msg("Ambari server upgrade failed. Please look at {0}, for more details.".format(configDefaults.SERVER_LOG_FILE))
raise FatalException(11, 'Schema upgrade failed.')
user = read_ambari_user()
if user is None:
warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG
print_warning_msg(warn)
else:
adjust_directory_permissions(user)
# local repo
upgrade_local_repo(args)
# create jdbc symlinks if jdbc drivers are available in resources
check_jdbc_drivers(args)
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
# Move *.py files from custom_actions to custom_actions/scripts
# This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0
ambari_version = get_ambari_version(properties)
if ambari_version is None:
args.warnings.append("*.py files were not moved from custom_actions to custom_actions/scripts.")
elif compare_versions(ambari_version, "2.0.0") == 0:
move_user_custom_actions()
# Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0
admin_views_dirs = get_admin_views_dir(properties)
for admin_views_dir in admin_views_dirs:
shutil.rmtree(admin_views_dir)
# Modify timestamp of views jars to current time
views_jars = get_views_jars(properties)
for views_jar in views_jars:
os.utime(views_jar, None)
# check if ambari has obsolete LDAP configuration
if properties.get_property(LDAP_PRIMARY_URL_PROPERTY) and not properties.get_property(IS_LDAP_CONFIGURED):
args.warnings.append("Existing LDAP configuration is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration.")
# adding custom jdbc name and previous custom jdbc properties
# we need that to support new dynamic jdbc names for upgraded ambari
add_jdbc_properties(properties)
def server_process_main(options, scmStatus=None):
# debug mode, including stop Java process at startup
try:
set_debug_mode_from_options(options)
except AttributeError:
pass
if not check_reverse_lookup():
print_warning_msg(
"The hostname was not found in the reverse DNS lookup. "
"This may result in incorrect behavior. "
"Please check the DNS setup and fix the issue.")
check_database_name_property()
parse_properties_file(options)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
print_info_msg("Ambari Server is not running...")
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
properties = get_ambari_properties()
# Preparations
if is_root():
print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT
ensure_jdbc_driver_is_installed(options, properties)
ensure_dbms_is_running(options, properties, scmStatus)
if scmStatus is not None:
scmStatus.reportStartPending()
refresh_stack_hash(properties)
if scmStatus is not None:
scmStatus.reportStartPending()
ensure_server_security_is_configured()
if scmStatus is not None:
scmStatus.reportStartPending()
java_exe = get_java_exe_path()
class_path = get_conf_dir()
class_path = os.path.abspath(
class_path) + os.pathsep + get_ambari_classpath()
debug_mode = get_debug_mode()
debug_start = (debug_mode & 1) or SERVER_START_DEBUG
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
(param_list,
environ) = generate_child_process_param_list(ambari_user, current_user,
java_exe, class_path,
debug_start, suspend_mode)
if not os.path.exists(configDefaults.PID_DIR):
os.makedirs(configDefaults.PID_DIR, 0755)
print_info_msg("Running server: " + str(param_list))
procJava = subprocess.Popen(param_list, env=environ)
pidJava = procJava.pid
if pidJava <= 0:
procJava.terminate()
exitcode = procJava.returncode
exitfile = os.path.join(configDefaults.PID_DIR, EXITCODE_NAME)
save_pid(exitcode, exitfile)
if scmStatus is not None:
scmStatus.reportStopPending()
raise FatalException(
-1,
AMBARI_SERVER_DIE_MSG.format(exitcode,
configDefaults.SERVER_OUT_FILE))
else:
pidfile = os.path.join(configDefaults.PID_DIR, PID_NAME)
save_pid(pidJava, pidfile)
print "Server PID at: " + pidfile
print "Server out at: " + configDefaults.SERVER_OUT_FILE
print "Server log at: " + configDefaults.SERVER_LOG_FILE
wait_for_server_start(pidfile, scmStatus)
if scmStatus is not None:
scmStatus.reportStarted()
return procJava
def generate_child_process_param_list(ambari_user, current_user, java_exe,
class_path, debug_start, suspend_mode):
from ambari_commons.os_linux import ULIMIT_CMD
properties = get_ambari_properties()
isSecure = get_is_secure(properties)
(isPersisted, masterKeyFile) = get_is_persisted(properties)
environ = os.environ.copy()
# Need to handle master key not persisted scenario
if isSecure and not masterKeyFile:
prompt = False
masterKey = environ.get(SECURITY_KEY_ENV_VAR_NAME)
if masterKey is not None and masterKey != "":
pass
else:
keyLocation = environ.get(SECURITY_MASTER_KEY_LOCATION)
if keyLocation is not None:
try:
# Verify master key can be read by the java process
with open(keyLocation, 'r'):
pass
except IOError:
print_warning_msg(
"Cannot read Master key from path specified in "
"environemnt.")
prompt = True
else:
# Key not provided in the environment
prompt = True
if prompt:
import pwd
masterKey = get_original_master_key(properties)
tempDir = tempfile.gettempdir()
tempFilePath = tempDir + os.sep + "masterkey"
save_master_key(masterKey, tempFilePath, True)
if ambari_user != current_user:
uid = pwd.getpwnam(ambari_user).pw_uid
gid = pwd.getpwnam(ambari_user).pw_gid
os.chown(tempFilePath, uid, gid)
else:
os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE)
if tempFilePath is not None:
environ[SECURITY_MASTER_KEY_LOCATION] = tempFilePath
command_base = SERVER_START_CMD_DEBUG if debug_start else SERVER_START_CMD
ulimit_cmd = "%s %s" % (ULIMIT_CMD, str(get_ulimit_open_files(properties)))
command = command_base.format(
java_exe, ambari_provider_module_option, jvm_args, class_path,
configDefaults.SERVER_OUT_FILE,
os.path.join(configDefaults.PID_DIR, EXITCODE_NAME), suspend_mode)
# required to start properly server instance
os.chdir(configDefaults.ROOT_FS_PATH)
#For properly daemonization server should be started using shell as parent
param_list = [locate_file('sh', '/bin'), "-c"]
if is_root() and ambari_user != "root":
# To inherit exported environment variables (especially AMBARI_PASSPHRASE),
# from subprocess, we have to skip --login option of su command. That's why
# we change dir to / (otherwise subprocess can face with 'permission denied'
# errors while trying to list current directory
cmd = "{ulimit_cmd} ; {su} {ambari_user} -s {sh_shell} -c '{command}'".format(
ulimit_cmd=ulimit_cmd,
su=locate_file('su', '/bin'),
ambari_user=ambari_user,
sh_shell=locate_file('sh', '/bin'),
command=command)
else:
cmd = "{ulimit_cmd} ; {command}".format(ulimit_cmd=ulimit_cmd,
command=command)
param_list.append(cmd)
return (param_list, environ)
def refresh_stack_hash_action():
logger.info("Refresh stack hash.")
properties = get_ambari_properties()
refresh_stack_hash(properties)
def upgrade(args):
if not is_root():
err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT
raise FatalException(4, err)
print 'Updating properties in ' + AMBARI_PROPERTIES_FILE + ' ...'
retcode = update_ambari_properties()
if not retcode == 0:
err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
try:
update_database_name_property(upgrade=True)
except FatalException:
return -1
# Ignore the server version & database options passed via command-line arguments
parse_properties_file(args)
#TODO check database version
change_objects_owner(args)
retcode = run_schema_upgrade()
if not retcode == 0:
print_error_msg("Ambari server upgrade failed. Please look at {0}, for more details.".format(configDefaults.SERVER_LOG_FILE))
raise FatalException(11, 'Schema upgrade failed.')
user = read_ambari_user()
if user is None:
warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG
print_warning_msg(warn)
else:
adjust_directory_permissions(user)
# local repo
upgrade_local_repo(args)
# create jdbc symlinks if jdbc drivers are available in resources
check_jdbc_drivers(args)
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
# Move *.py files from custom_actions to custom_actions/scripts
# This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0
ambari_version = get_ambari_version(properties)
if ambari_version is None:
args.warnings.append("*.py files were not moved from custom_actions to custom_actions/scripts.")
elif compare_versions(ambari_version, "2.0.0") == 0:
move_user_custom_actions()
# Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0
admin_views_dirs = get_admin_views_dir(properties)
for admin_views_dir in admin_views_dirs:
shutil.rmtree(admin_views_dir)
# check if ambari has obsolete LDAP configuration
if properties.get_property(LDAP_PRIMARY_URL_PROPERTY) and not properties.get_property(IS_LDAP_CONFIGURED):
args.warnings.append("Existing LDAP configuration is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration.")
def set_current(options):
logger.info("Set current cluster version.")
server_status, pid = is_server_runing()
if not server_status:
err = 'Ambari Server is not running.'
raise FatalException(1, err)
finalize_options = SetCurrentVersionOptions(options)
if finalize_options.no_finalize_options_set():
err = 'Must specify --cluster-name and --version-display-name. Please invoke ambari-server.py --help to print the options.'
raise FatalException(1, err)
admin_login = get_validated_string_input(
prompt="Enter Ambari Admin login: "******"Enter Ambari Admin password: "******"Failed to read properties file.")
base_url = get_ambari_server_api_base(properties)
url = base_url + "clusters/{0}/stack_versions".format(
finalize_options.cluster_name)
admin_auth = base64.encodestring(
'%s:%s' % (admin_login, admin_password)).replace('\n', '')
request = urllib2.Request(url)
request.add_header('Authorization', 'Basic %s' % admin_auth)
request.add_header('X-Requested-By', 'ambari')
data = {
"ClusterStackVersions": {
"repository_version": finalize_options.desired_repo_version,
"state": "CURRENT",
"force": finalize_options.force_repo_version
}
}
if get_verbose():
sys.stdout.write('\nCalling API ' + url + ' : ' + str(data) + '\n')
request.add_data(json.dumps(data))
request.get_method = lambda: 'PUT'
try:
response = urllib2.urlopen(request)
except urllib2.HTTPError, e:
code = e.getcode()
content = e.read()
err = 'Error during setting current version. Http status code - {0}. \n {1}'.format(
code, content)
raise FatalException(1, err)
def refresh_stack_hash_action():
properties = get_ambari_properties()
refresh_stack_hash(properties)
if not stack_version:
Logger.error(
"Could not parse HDP version from output of hdp-select: %s"
% str(out))
return 1
else:
stack_version = options.hdp_version
return stack_version
parser = OptionParser()
parser.add_option("-d",
"--database-driver",
dest="sql_driver_path",
default=get_value_from_properties(
get_ambari_properties(), JDBC_DRIVER_PATH_PROPERTY,
DEFAULT_SQL_DRIVER_PATH),
help="Path to JDBC driver")
parser.add_option("-f",
"--fs-type",
dest="fs_type",
default="wasb",
help="Expected protocol of fs.defaultFS")
parser.add_option("-v",
"--hdp-version",
dest="hdp_version",
default="",
help="hdp-version used in path of tarballs")
parser.add_option("-u",
"--upgrade",
dest="upgrade",
def setup_sso(args):
if not is_root():
err = 'ambari-server setup-sso should be run with ' \
'root-level privileges'
raise FatalException(4, err)
if not get_silent():
properties = get_ambari_properties()
must_setup_params = False
store_new_cert = False
sso_enabled = properties.get_property(JWT_AUTH_ENBABLED).lower() in [
'true'
]
if sso_enabled:
if get_YN_input(
"Do you want to disable SSO authentication [y/n] (n)?",
False):
properties.process_pair(JWT_AUTH_ENBABLED, "false")
else:
if get_YN_input(
"Do you want to configure SSO authentication [y/n] (y)?",
True):
properties.process_pair(JWT_AUTH_ENBABLED, "true")
must_setup_params = True
else:
return False
if must_setup_params:
provider_url = get_value_from_properties(
properties, JWT_AUTH_PROVIDER_URL,
JWT_AUTH_PROVIDER_URL_DEFAULT)
provider_url = get_validated_string_input(
"Provider URL [URL] ({0}):".format(provider_url), provider_url,
REGEX_ANYTHING, "Invalid provider URL", False)
properties.process_pair(JWT_AUTH_PROVIDER_URL, provider_url)
cert_path = properties.get_property(JWT_PUBLIC_KEY)
cert_string = get_multi_line_input(
"Public Certificate pem ({0})".format(
'stored' if cert_path else 'empty'))
if cert_string is not None:
store_new_cert = True
if get_YN_input(
"Do you want to configure advanced properties [y/n] (n) ?",
False):
cookie_name = get_value_from_properties(
properties, JWT_COOKIE_NAME, JWT_COOKIE_NAME_DEFAULT)
cookie_name = get_validated_string_input(
"JWT Cookie name ({0}):".format(cookie_name), cookie_name,
REGEX_ANYTHING, "Invalid cookie name", False)
properties.process_pair(JWT_COOKIE_NAME, cookie_name)
audiences = properties.get_property(JWT_AUDIENCES)
audiences = get_validated_string_input(
"JWT audiences list (comma-separated), empty for any ({0}):"
.format(audiences), audiences, REGEX_ANYTHING,
"Invalid value", False)
properties.process_pair(JWT_AUDIENCES, audiences)
# TODO not required for now as we support Knox only
# orig_query_param = get_value_from_properties(JWT_ORIGINAL_URL_QUERY_PARAM, JWT_ORIGINAL_URL_QUERY_PARAM_DEFAULT)
# orig_query_param = get_validated_string_input("Original URL query parameter name ({}):".format(orig_query_param),
# orig_query_param,
# REGEX_ANYTHING,
# "Invalid value",
# False)
# properties.process_pair(JWT_ORIGINAL_URL_QUERY_PARAM, orig_query_param)
if store_new_cert:
full_cert = JWT_PUBLIC_KEY_HEADER + cert_string + JWT_PUBLIC_KEY_FOOTER
cert_path = store_password_file(full_cert,
JWT_PUBLIC_KEY_FILENAME)
properties.process_pair(JWT_PUBLIC_KEY, cert_path)
update_properties(properties)
pass
else:
warning = "setup-sso is not enabled in silent mode."
raise NonFatalException(warning)
pass
def getServiceConfigurationRecommendations(self, configurations,
clusterData, services, hosts):
Logger.info(
"Class: %s, Method: %s. Get Service Configuration Recommendations."
% (self.__class__.__name__, inspect.stack()[0][3]))
servicesList = [
service["StackServices"]["service_name"]
for service in services["services"]
]
security_enabled = self.isSecurityEnabled(services)
if 'AMBARI_METRICS' in servicesList:
putAmsSiteProperty = self.putProperty(configurations, "ams-site")
putAmsSiteProperty(
'timeline.metrics.downsampler.event.metric.patterns',
'topology\.%')
if 'STORM' in servicesList and security_enabled:
storm_site = self.getServicesSiteProperties(services, "storm-site")
streamline_env = self.getServicesSiteProperties(
services, "streamline-env")
if storm_site is not None and streamline_env is not None:
putStormSiteProperty = self.putProperty(
configurations, "storm-site", services)
putStormSiteAttributes = self.putPropertyAttribute(
configurations, "storm-site")
storm_env = self.getServicesSiteProperties(
services, "storm-env")
storm_nimbus_impersonation_acl = storm_site[
"nimbus.impersonation.acl"] if "nimbus.impersonation.acl" in storm_site else None
streamline_env = self.getServicesSiteProperties(
services, "streamline-env")
_streamline_principal_name = streamline_env[
'streamline_principal_name'] if 'streamline_principal_name' in streamline_env else None
if _streamline_principal_name is not None and storm_nimbus_impersonation_acl is not None:
streamline_bare_principal = get_bare_principal(
_streamline_principal_name)
storm_nimbus_impersonation_acl = "{ " + streamline_bare_principal + " : {hosts: ['*'], groups: ['*']}, {{storm_bare_jaas_principal}} : {hosts: ['*'], groups: ['*']}}"
putStormSiteProperty('nimbus.impersonation.acl',
storm_nimbus_impersonation_acl)
storm_nimbus_autocred_plugin_classes = storm_site[
"nimbus.autocredential.plugins.classes"] if "nimbus.autocredential.plugins.classes" in storm_site else None
# Here storm_nimbus_autocred_plugin_classes is resulting in none. There is no nimbus.autocredential.plugins.classes in storm-site.xml
if storm_nimbus_autocred_plugin_classes is not None:
new_storm_nimbus_autocred_plugin_classes = [
'org.apache.storm.hdfs.security.AutoHDFS',
'org.apache.storm.hbase.security.AutoHBase',
'org.apache.storm.hive.security.AutoHive'
]
new_conf = self.appendToYamlString(
storm_nimbus_autocred_plugin_classes,
new_storm_nimbus_autocred_plugin_classes)
putStormSiteProperty(
"nimbus.autocredential.plugins.classes", new_conf)
else:
putStormSiteProperty(
"nimbus.autocredential.plugins.classes",
"['org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive']"
)
storm_nimbus_credential_renewer_classes = storm_site[
"nimbus.credential.renewers.classes"] if "nimbus.credential.renewers.classes" in storm_site else None
if storm_nimbus_credential_renewer_classes is not None:
new_storm_nimbus_credential_renewer_classes_array = [
'org.apache.storm.hdfs.security.AutoHDFS',
'org.apache.storm.hbase.security.AutoHBase',
'org.apache.storm.hive.security.AutoHive'
]
new_conf = self.appendToYamlString(
storm_nimbus_credential_renewer_classes,
new_storm_nimbus_credential_renewer_classes_array)
putStormSiteProperty(
"nimbus.autocredential.plugins.classes", new_conf)
else:
putStormSiteProperty(
"nimbus.credential.renewers.classes",
"['org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive']"
)
putStormSiteProperty("nimbus.credential.renewers.freq.secs",
"82800")
properties = get_ambari_properties()
ambari_version = get_ambari_version(properties)
if not (ambari_version) or not (ambari_version.startswith('2.5')):
putStreamlineLogSearchConfAttribute = self.putPropertyAttribute(
configurations, "streamline-logsearch-conf")
putStreamlineLogSearchConfAttribute('service_name', 'visible',
'false')
putStreamlineLogSearchConfAttribute('component_mappings',
'visible', 'false')
putStreamlineLogSearchConfAttribute('content', 'visible', 'false')
pass
def setup_ldap(options):
logger.info("Setup LDAP.")
properties = get_ambari_properties()
server_status, pid = is_server_runing()
if not server_status:
err = 'Ambari Server is not running.'
raise FatalException(1, err)
current_client_security = get_value_from_properties(properties,CLIENT_SECURITY,"no auth method")
if current_client_security != 'ldap':
query = "Currently '" + current_client_security + "' is configured, do you wish to use LDAP instead [y/n] (n)? "
if get_YN_input(query, False):
pass
else:
err = "Currently '" + current_client_security + "' configured. Can not setup LDAP."
raise FatalException(1, err)
isSecure = get_is_secure(properties)
ldap_property_list_reqd = init_ldap_properties_list_reqd(properties, options)
ldap_property_list_opt = [LDAP_MGR_USERNAME_PROPERTY,
LDAP_MGR_PASSWORD_PROPERTY,
SSL_TRUSTSTORE_TYPE_PROPERTY,
SSL_TRUSTSTORE_PATH_PROPERTY,
SSL_TRUSTSTORE_PASSWORD_PROPERTY]
ldap_property_list_passwords=[LDAP_MGR_PASSWORD_PROPERTY,
SSL_TRUSTSTORE_PASSWORD_PROPERTY]
LDAP_MGR_DN_DEFAULT = None
SSL_TRUSTSTORE_TYPE_DEFAULT = get_value_from_properties(properties, SSL_TRUSTSTORE_TYPE_PROPERTY, "jks")
SSL_TRUSTSTORE_PATH_DEFAULT = get_value_from_properties(properties, SSL_TRUSTSTORE_PATH_PROPERTY)
ldap_property_value_map = {}
for ldap_prop in ldap_property_list_reqd:
input = get_validated_string_input(ldap_prop.ldap_prop_val_prompt, ldap_prop.ldap_prop_name, ldap_prop.prompt_regex,
"Invalid characters in the input!", False, ldap_prop.allow_empty_prompt,
answer = ldap_prop.option)
if input is not None and input != "":
ldap_property_value_map[ldap_prop.prop_name] = input
bindAnonymously = ldap_property_value_map[LDAP_ANONYMOUS_BIND]
anonymous = (bindAnonymously and bindAnonymously.lower() == 'true')
mgr_password = None
# Ask for manager credentials only if bindAnonymously is false
if not anonymous:
username = get_validated_string_input("Manager DN* {0}: ".format(
get_prompt_default(LDAP_MGR_DN_DEFAULT)), LDAP_MGR_DN_DEFAULT, ".*",
"Invalid characters in the input!", False, False, answer = options.ldap_manager_dn)
ldap_property_value_map[LDAP_MGR_USERNAME_PROPERTY] = username
mgr_password = configure_ldap_password(options)
ldap_property_value_map[LDAP_MGR_PASSWORD_PROPERTY] = mgr_password
useSSL = ldap_property_value_map[LDAP_USE_SSL]
ldaps = (useSSL and useSSL.lower() == 'true')
ts_password = None
if ldaps:
truststore_default = "n"
truststore_set = bool(SSL_TRUSTSTORE_PATH_DEFAULT)
if truststore_set:
truststore_default = "y"
custom_trust_store = True if options.trust_store_path is not None and options.trust_store_path else False
if not custom_trust_store:
custom_trust_store = get_YN_input("Do you want to provide custom TrustStore for Ambari [y/n] ({0})?".
format(truststore_default),
truststore_set)
if custom_trust_store:
ts_type = get_validated_string_input("TrustStore type [jks/jceks/pkcs12] {0}:".format(get_prompt_default(SSL_TRUSTSTORE_TYPE_DEFAULT)),
SSL_TRUSTSTORE_TYPE_DEFAULT, "^(jks|jceks|pkcs12)?$", "Wrong type", False, answer=options.trust_store_type)
ts_path = None
while True:
ts_path = get_validated_string_input("Path to TrustStore file {0}:".format(get_prompt_default(SSL_TRUSTSTORE_PATH_DEFAULT)),
SSL_TRUSTSTORE_PATH_DEFAULT, ".*", False, False, answer = options.trust_store_path)
if os.path.exists(ts_path):
break
else:
print 'File not found.'
hasAnswer = options.trust_store_path is not None and options.trust_store_path
quit_if_has_answer(hasAnswer)
ts_password = read_password("", ".*", "Password for TrustStore:", "Invalid characters in password", options.trust_store_password)
ldap_property_value_map[SSL_TRUSTSTORE_TYPE_PROPERTY] = ts_type
ldap_property_value_map[SSL_TRUSTSTORE_PATH_PROPERTY] = ts_path
ldap_property_value_map[SSL_TRUSTSTORE_PASSWORD_PROPERTY] = ts_password
pass
elif properties.get_property(SSL_TRUSTSTORE_TYPE_PROPERTY):
print 'The TrustStore is already configured: '
print ' ' + SSL_TRUSTSTORE_TYPE_PROPERTY + ' = ' + properties.get_property(SSL_TRUSTSTORE_TYPE_PROPERTY)
print ' ' + SSL_TRUSTSTORE_PATH_PROPERTY + ' = ' + properties.get_property(SSL_TRUSTSTORE_PATH_PROPERTY)
print ' ' + SSL_TRUSTSTORE_PASSWORD_PROPERTY + ' = ' + properties.get_property(SSL_TRUSTSTORE_PASSWORD_PROPERTY)
if get_YN_input("Do you want to remove these properties [y/n] (y)? ", True, options.trust_store_reconfigure):
properties.removeOldProp(SSL_TRUSTSTORE_TYPE_PROPERTY)
properties.removeOldProp(SSL_TRUSTSTORE_PATH_PROPERTY)
properties.removeOldProp(SSL_TRUSTSTORE_PASSWORD_PROPERTY)
pass
pass
print '=' * 20
print 'Review Settings'
print '=' * 20
for property in ldap_property_list_reqd:
if ldap_property_value_map.has_key(property):
print("%s: %s" % (property, ldap_property_value_map[property]))
for property in ldap_property_list_opt:
if ldap_property_value_map.has_key(property):
if property not in ldap_property_list_passwords:
print("%s: %s" % (property, ldap_property_value_map[property]))
else:
print("%s: %s" % (property, BLIND_PASSWORD))
save_settings = True if options.ldap_save_settings is not None else get_YN_input("Save settings [y/n] (y)? ", True)
if save_settings:
if isSecure:
if mgr_password:
encrypted_passwd = encrypt_password(LDAP_MGR_PASSWORD_ALIAS, mgr_password, options)
if mgr_password != encrypted_passwd:
ldap_property_value_map[LDAP_MGR_PASSWORD_PROPERTY] = encrypted_passwd
pass
if ts_password:
encrypted_passwd = encrypt_password(SSL_TRUSTSTORE_PASSWORD_ALIAS, ts_password, options)
if ts_password != encrypted_passwd:
ldap_property_value_map[SSL_TRUSTSTORE_PASSWORD_PROPERTY] = encrypted_passwd
pass
pass
# Persisting values
if mgr_password:
ldap_property_value_map[LDAP_MGR_PASSWORD_PROPERTY] = store_password_file(mgr_password, LDAP_MGR_PASSWORD_FILENAME)
print 'Saving LDAP properties...'
ldap_property_value_map[IS_LDAP_CONFIGURED] = "true"
#Saving LDAP configuration in Ambari DB using the REST API
update_ldap_configuration(properties, ldap_property_value_map)
#The only property we want to write out in Ambari.properties is the client.security type being LDAP
ldap_property_value_map.clear()
ldap_property_value_map[CLIENT_SECURITY] = 'ldap'
update_properties_2(properties, ldap_property_value_map)
print 'Saving LDAP properties finished'
return 0
def setup_sso(options):
logger.info("Setup SSO.")
if not is_root():
raise FatalException(
4,
'ambari-server setup-sso should be run with root-level privileges')
server_status, pid = is_server_runing()
if not server_status:
err = 'Ambari Server is not running.'
raise FatalException(1, err)
if not get_silent():
validate_options(options)
properties = get_ambari_properties()
admin_login, admin_password = get_ambari_admin_username_password_pair(
options)
if not options.sso_enabled:
sso_enabled_from_db = get_sso_property_from_db(
properties, admin_login, admin_password, SSO_MANAGE_SERVICES)
sso_enabled = sso_enabled_from_db == None or sso_enabled_from_db in [
'true'
]
sys.stdout.write("\nSSO is currently {0}\n".format(
"not configured" if sso_enabled_from_db == None else (
"enabled" if sso_enabled else "disabled")))
if sso_enabled:
enable_sso = not get_YN_input(
"Do you want to disable SSO authentication [y/n] (n)? ",
False)
else:
if get_YN_input(
"Do you want to configure SSO authentication [y/n] (y)? ",
True):
enable_sso = True
else:
return False
else:
enable_sso = options.sso_enabled == 'true'
services = ''
if enable_sso:
populate_sso_provider_url(options, properties)
populate_sso_public_cert(options, properties)
populate_jwt_cookie_name(options, properties)
populate_jwt_audiences(options, properties)
services = get_services_requires_sso(options, properties,
admin_login, admin_password)
update_sso_conf(properties, enable_sso, services, admin_login,
admin_password)
enable_jwt_auth = WILDCARD_FOR_ALL_SERVICES == services or SERVICE_NAME_AMBARI in services
properties.process_pair(JWT_AUTH_ENBABLED,
"true" if enable_jwt_auth else "false")
update_properties(properties)
pass
else:
warning = "setup-sso is not enabled in silent mode."
raise NonFatalException(warning)
pass
def server_process_main(options, scmStatus=None):
# debug mode, including stop Java process at startup
try:
set_debug_mode_from_options(options)
except AttributeError:
pass
if not check_reverse_lookup():
print_warning_msg("The hostname was not found in the reverse DNS lookup. "
"This may result in incorrect behavior. "
"Please check the DNS setup and fix the issue.")
check_database_name_property()
parse_properties_file(options)
is_active_instance = get_is_active_instance()
if not is_active_instance:
print_warning_msg("This instance of ambari server is not designated as active. Cannot start ambari server.")
err = "This is not an active instance. Shutting down..."
raise FatalException(1, err)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
print_info_msg("Ambari Server is not running...")
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
properties = get_ambari_properties()
# Preparations
if is_root():
print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT
ensure_jdbc_driver_is_installed(options, properties)
ensure_dbms_is_running(options, properties, scmStatus)
if scmStatus is not None:
scmStatus.reportStartPending()
refresh_stack_hash(properties)
if scmStatus is not None:
scmStatus.reportStartPending()
ensure_server_security_is_configured()
if scmStatus is not None:
scmStatus.reportStartPending()
java_exe = get_java_exe_path()
class_path = get_conf_dir()
class_path = os.path.abspath(class_path) + os.pathsep + get_ambari_classpath()
jdbc_driver_path = get_jdbc_driver_path(options, properties)
if jdbc_driver_path not in class_path:
class_path = class_path + os.pathsep + jdbc_driver_path
if SERVER_CLASSPATH_KEY in os.environ:
class_path = os.environ[SERVER_CLASSPATH_KEY] + os.pathsep + class_path
native_libs_path = get_native_libs_path(options, properties)
if native_libs_path is not None:
if LIBRARY_PATH_KEY in os.environ:
native_libs_path = os.environ[LIBRARY_PATH_KEY] + os.pathsep + native_libs_path
os.environ[LIBRARY_PATH_KEY] = native_libs_path
debug_mode = get_debug_mode()
debug_start = (debug_mode & 1) or SERVER_START_DEBUG
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
param_list = generate_child_process_param_list(ambari_user, java_exe,
class_path, debug_start,
suspend_mode)
environ = generate_env(ambari_user, current_user)
if not os.path.exists(configDefaults.PID_DIR):
os.makedirs(configDefaults.PID_DIR, 0755)
print_info_msg("Running server: " + str(param_list))
procJava = subprocess.Popen(param_list, env=environ)
pidJava = procJava.pid
if pidJava <= 0:
procJava.terminate()
exitcode = procJava.returncode
exitfile = os.path.join(configDefaults.PID_DIR, EXITCODE_NAME)
save_pid(exitcode, exitfile)
if scmStatus is not None:
scmStatus.reportStopPending()
raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, configDefaults.SERVER_OUT_FILE))
else:
# Change the group id to the process id of the parent so that the launched
# process and sub-processes have a group id that is different from the parent.
try:
os.setpgid(pidJava, 0)
except OSError, e:
print_warning_msg('setpgid({0}, 0) failed - {1}'.format(pidJava, str(e)))
pass
pidfile = os.path.join(configDefaults.PID_DIR, PID_NAME)
save_pid(pidJava, pidfile)
print "Server PID at: "+pidfile
print "Server out at: "+configDefaults.SERVER_OUT_FILE
print "Server log at: "+configDefaults.SERVER_LOG_FILE
wait_for_server_start(pidfile, scmStatus)
def setup_ldap():
if not is_root():
err = 'Ambari-server setup-ldap should be run with ' \
'root-level privileges'
raise FatalException(4, err)
properties = get_ambari_properties()
isSecure = get_is_secure(properties)
ldap_property_list_reqd = init_ldap_properties_list_reqd(properties)
ldap_property_list_opt = ["authentication.ldap.managerDn",
LDAP_MGR_PASSWORD_PROPERTY,
SSL_TRUSTSTORE_TYPE_PROPERTY,
SSL_TRUSTSTORE_PATH_PROPERTY,
SSL_TRUSTSTORE_PASSWORD_PROPERTY]
ldap_property_list_truststore=[SSL_TRUSTSTORE_TYPE_PROPERTY,
SSL_TRUSTSTORE_PATH_PROPERTY,
SSL_TRUSTSTORE_PASSWORD_PROPERTY]
ldap_property_list_passwords=[LDAP_MGR_PASSWORD_PROPERTY,
SSL_TRUSTSTORE_PASSWORD_PROPERTY]
LDAP_MGR_DN_DEFAULT = get_value_from_properties(properties, ldap_property_list_opt[0])
SSL_TRUSTSTORE_TYPE_DEFAULT = get_value_from_properties(properties, SSL_TRUSTSTORE_TYPE_PROPERTY, "jks")
SSL_TRUSTSTORE_PATH_DEFAULT = get_value_from_properties(properties, SSL_TRUSTSTORE_PATH_PROPERTY)
ldap_property_value_map = {}
for ldap_prop in ldap_property_list_reqd:
input = get_validated_string_input(ldap_prop.ldap_prop_val_prompt, ldap_prop.ldap_prop_name, ldap_prop.prompt_regex,
"Invalid characters in the input!", False, ldap_prop.allow_empty_prompt)
if input is not None and input != "":
ldap_property_value_map[ldap_prop.prop_name] = input
bindAnonymously = ldap_property_value_map["authentication.ldap.bindAnonymously"]
anonymous = (bindAnonymously and bindAnonymously.lower() == 'true')
mgr_password = None
# Ask for manager credentials only if bindAnonymously is false
if not anonymous:
username = get_validated_string_input("Manager DN* {0}: ".format(
get_prompt_default(LDAP_MGR_DN_DEFAULT)), LDAP_MGR_DN_DEFAULT, ".*",
"Invalid characters in the input!", False, False)
ldap_property_value_map[LDAP_MGR_USERNAME_PROPERTY] = username
mgr_password = configure_ldap_password()
ldap_property_value_map[LDAP_MGR_PASSWORD_PROPERTY] = mgr_password
useSSL = ldap_property_value_map["authentication.ldap.useSSL"]
ldaps = (useSSL and useSSL.lower() == 'true')
ts_password = None
if ldaps:
truststore_default = "n"
truststore_set = bool(SSL_TRUSTSTORE_PATH_DEFAULT)
if truststore_set:
truststore_default = "y"
custom_trust_store = get_YN_input("Do you want to provide custom TrustStore for Ambari [y/n] ({0})?".
format(truststore_default),
truststore_set)
if custom_trust_store:
ts_type = get_validated_string_input(
"TrustStore type [jks/jceks/pkcs12] {0}:".format(get_prompt_default(SSL_TRUSTSTORE_TYPE_DEFAULT)),
SSL_TRUSTSTORE_TYPE_DEFAULT,
"^(jks|jceks|pkcs12)?$", "Wrong type", False)
ts_path = None
while True:
ts_path = get_validated_string_input(
"Path to TrustStore file {0}:".format(get_prompt_default(SSL_TRUSTSTORE_PATH_DEFAULT)),
SSL_TRUSTSTORE_PATH_DEFAULT,
".*", False, False)
if os.path.exists(ts_path):
break
else:
print 'File not found.'
ts_password = read_password("", ".*", "Password for TrustStore:", "Invalid characters in password")
ldap_property_value_map[SSL_TRUSTSTORE_TYPE_PROPERTY] = ts_type
ldap_property_value_map[SSL_TRUSTSTORE_PATH_PROPERTY] = ts_path
ldap_property_value_map[SSL_TRUSTSTORE_PASSWORD_PROPERTY] = ts_password
pass
else:
properties.removeOldProp(SSL_TRUSTSTORE_TYPE_PROPERTY)
properties.removeOldProp(SSL_TRUSTSTORE_PATH_PROPERTY)
properties.removeOldProp(SSL_TRUSTSTORE_PASSWORD_PROPERTY)
pass
pass
print '=' * 20
print 'Review Settings'
print '=' * 20
for property in ldap_property_list_reqd:
if property in ldap_property_value_map:
print("%s: %s" % (property, ldap_property_value_map[property]))
for property in ldap_property_list_opt:
if ldap_property_value_map.has_key(property):
if property not in ldap_property_list_passwords:
print("%s: %s" % (property, ldap_property_value_map[property]))
else:
print("%s: %s" % (property, BLIND_PASSWORD))
save_settings = get_YN_input("Save settings [y/n] (y)? ", True)
if save_settings:
ldap_property_value_map[CLIENT_SECURITY_KEY] = 'ldap'
if isSecure:
if mgr_password:
encrypted_passwd = encrypt_password(LDAP_MGR_PASSWORD_ALIAS, mgr_password)
if mgr_password != encrypted_passwd:
ldap_property_value_map[LDAP_MGR_PASSWORD_PROPERTY] = encrypted_passwd
pass
if ts_password:
encrypted_passwd = encrypt_password(SSL_TRUSTSTORE_PASSWORD_ALIAS, ts_password)
if ts_password != encrypted_passwd:
ldap_property_value_map[SSL_TRUSTSTORE_PASSWORD_PROPERTY] = encrypted_passwd
pass
pass
# Persisting values
ldap_property_value_map[IS_LDAP_CONFIGURED] = "true"
if mgr_password:
ldap_property_value_map[LDAP_MGR_PASSWORD_PROPERTY] = store_password_file(mgr_password, LDAP_MGR_PASSWORD_FILENAME)
update_properties_2(properties, ldap_property_value_map)
print 'Saving...done'
return 0
def wait_for_server_start(pidFile, scmStatus):
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
raise FatalException(-1, err)
#wait for server process for SERVER_START_TIMEOUT seconds
sys.stdout.write('Waiting for server start...')
sys.stdout.flush()
pids = []
pid = None
# looking_for_pid() might return partrial pid list on slow hardware
for i in range(1, SERVER_START_RETRIES):
pids = looking_for_pid(SERVER_SEARCH_PATTERN, SERVER_START_TIMEOUT)
pid = save_main_pid_ex(
pids, pidFile,
locate_all_file_paths('sh', '/bin') +
locate_all_file_paths('bash', '/bin') +
locate_all_file_paths('dash', '/bin'), IS_FOREGROUND)
if pid:
break
else:
sys.stdout.write("Unable to determine server PID. Retrying...\n")
sys.stdout.flush()
exception = None
if pid:
ambari_server_ui_port = get_ambari_server_ui_port(properties)
web_server_startup_timeout = get_web_server_startup_timeout(properties)
waitStart = time.time()
if not wait_for_ui_start(int(ambari_server_ui_port), pid,
web_server_startup_timeout):
waitTime = int(time.time() - waitStart)
# Java process stopped, due to a DB check or other startup issue
if waitTime < web_server_startup_timeout:
exception = FatalException(-1, AMBARI_SERVER_STOPPED)
# UI didn't come up on time
else:
exception = FatalException(
1,
AMBARI_SERVER_UI_TIMEOUT.format(
ambari_server_ui_port, web_server_startup_timeout))
elif get_live_pids_count(pids) <= 0:
exitcode = check_exitcode(
os.path.join(configDefaults.PID_DIR, EXITCODE_NAME))
exception = FatalException(
-1,
AMBARI_SERVER_DIE_MSG.format(exitcode,
configDefaults.SERVER_OUT_FILE))
else:
exception = FatalException(-1, AMBARI_SERVER_NOT_STARTED_MSG)
if os.path.isfile(configDefaults.SERVER_OUT_FILE):
if 'DB_CHECK_ERROR' in open(configDefaults.SERVER_OUT_FILE).read():
print "\nDB configs consistency check failed. Run \"ambari-server start --skip-database-check\" to skip. " \
"You may try --auto-fix-database flag to attempt to fix issues automatically. " \
"If you use this \"--skip-database-check\" option, do not make any changes to your cluster topology " \
"or perform a cluster upgrade until you correct the database consistency issues. See " + \
configDefaults.DB_CHECK_LOG + " for more details on the consistency issues."
elif 'DB_CHECK_WARNING' in open(configDefaults.SERVER_OUT_FILE).read():
print "\nDB configs consistency check found warnings. See " + configDefaults.DB_CHECK_LOG + " for more details."
# Only presume that DB check was successful if it explicitly appears in the log. An unexpected error may prevent
# the consistency check from running at all, so missing error/warning message in the log cannot imply the check was
# successful
elif 'DB_CHECK_SUCCESS' in open(configDefaults.SERVER_OUT_FILE).read():
print "\nDB configs consistency check: no errors and warnings were found."
else:
sys.stdout.write(configDefaults.SERVER_OUT_FILE + " does not exist")
if exception:
raise exception
def setup_truststore(import_cert=False):
if not get_silent():
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
properties = get_ambari_properties()
if get_YN_input("Do you want to configure a truststore [y/n] (y)? ",
True):
#Re-configuration enabled only for option "Setup truststore"
if not import_cert and properties.get_property(SSL_TRUSTSTORE_TYPE_PROPERTY)\
and get_YN_input(
"The truststore is already configured. Do you want to re-configure "
"the truststore [y/n] (y)? ", True):
properties.removeProp(SSL_TRUSTSTORE_TYPE_PROPERTY)
properties.removeProp(SSL_TRUSTSTORE_PATH_PROPERTY)
properties.removeProp(SSL_TRUSTSTORE_PASSWORD_PROPERTY)
truststore_type = get_and_persist_truststore_type(properties)
truststore_path = get_and_persist_truststore_path(properties)
truststore_password = get_and_persist_truststore_password(
properties)
if import_cert:
if get_YN_input(
"Do you want to import a certificate [y/n] (y)? ",
True):
alias = get_validated_string_input(
"Please enter an alias for the certificate: ", "",
None, None, False, False)
run_os_command(
get_delete_cert_command(jdk_path, alias,
truststore_path,
truststore_password))
import_cert_path = get_validated_filepath_input( \
"Enter path to certificate: ", \
"Certificate not found")
run_component_https_cmd(
get_import_cert_command(jdk_path, alias,
truststore_type,
import_cert_path,
truststore_path,
truststore_password))
else:
return
conf_file = find_properties_file()
f = open(conf_file, 'w')
properties.store(f,
"Changed by 'ambari-server setup-security' command")
else:
print "setup-security is not enabled in silent mode."
def run_schema_upgrade(args):
db_title = get_db_type(get_ambari_properties()).title
silent = get_silent()
default_answer = 'y' if silent else 'n'
default_value = silent
confirm = get_YN_input(
"Ambari Server configured for %s. Confirm "
"you have made a backup of the Ambari Server database [y/n] (%s)? " %
(db_title, default_answer), default_value)
if not confirm:
print_error_msg("Database backup is not confirmed")
return 1
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg(
"No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
ensure_jdbc_driver_is_installed(args, get_ambari_properties())
print_info_msg('Upgrading database schema', True)
serverClassPath = ServerClassPath(get_ambari_properties(), args)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell(
validate_classpath=True)
set_debug_mode_from_options(args)
debug_mode = get_debug_mode()
debug_start = (debug_mode & 1) or SCHEMA_UPGRADE_DEBUG
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
command = SCHEMA_UPGRADE_HELPER_CMD_DEBUG.format(
jdk_path, class_path,
suspend_mode) if debug_start else SCHEMA_UPGRADE_HELPER_CMD.format(
jdk_path, class_path)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
environ = generate_env(args, ambari_user, current_user)
(retcode, stdout, stderr) = run_os_command(command, env=environ)
upgrade_response = json.loads(stdout)
check_gpl_license_approved(upgrade_response)
print_info_msg(
"Return code from schema upgrade command, retcode = {0}".format(
str(retcode)), True)
if stdout:
print_info_msg("Console output from schema upgrade command:", True)
print_info_msg(stdout, True)
print
if retcode > 0:
print_error_msg(
"Error executing schema upgrade, please check the server logs.")
if stderr:
print_error_msg("Error output from schema upgrade command:")
print_error_msg(stderr)
print
else:
print_info_msg('Schema upgrade completed', True)
return retcode
def server_process_main(options, scmStatus=None):
# debug mode, including stop Java process at startup
try:
set_debug_mode_from_options(options)
except AttributeError:
pass
if not check_reverse_lookup():
print_warning_msg("The hostname was not found in the reverse DNS lookup. "
"This may result in incorrect behavior. "
"Please check the DNS setup and fix the issue.")
check_database_name_property()
parse_properties_file(options)
ambari_user = read_ambari_user()
current_user = ensure_can_start_under_current_user(ambari_user)
print_info_msg("Ambari Server is not running...")
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
"JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
properties = get_ambari_properties()
# Preparations
if is_root():
print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT
ensure_jdbc_driver_is_installed(options, properties)
ensure_dbms_is_running(options, properties, scmStatus)
if scmStatus is not None:
scmStatus.reportStartPending()
refresh_stack_hash(properties)
if scmStatus is not None:
scmStatus.reportStartPending()
ensure_server_security_is_configured()
if scmStatus is not None:
scmStatus.reportStartPending()
java_exe = get_java_exe_path()
serverClassPath = ServerClassPath(properties, options)
debug_mode = get_debug_mode()
debug_start = (debug_mode & 1) or SERVER_START_DEBUG
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
param_list = generate_child_process_param_list(ambari_user, java_exe,
serverClassPath.get_full_ambari_classpath_escaped_for_shell(), debug_start,
suspend_mode)
environ = generate_env(options, ambari_user, current_user)
if not os.path.exists(configDefaults.PID_DIR):
os.makedirs(configDefaults.PID_DIR, 0755)
# The launched shell process and sub-processes should have a group id that
# is different from the parent.
def make_process_independent():
if IS_FOREGROUND: # upstart script is not able to track process from different pgid.
return
processId = os.getpid()
if processId > 0:
try:
os.setpgid(processId, processId)
except OSError, e:
print_warning_msg('setpgid({0}, {0}) failed - {1}'.format(pidJava, str(e)))
pass
def upgrade(args):
print_info_msg("Upgrade Ambari Server", True)
if not is_root():
err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT
raise FatalException(4, err)
print_info_msg(
'Updating Ambari Server properties in {0} ...'.format(
AMBARI_PROPERTIES_FILE), True)
retcode = update_ambari_properties()
if not retcode == 0:
err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
print_info_msg(
'Updating Ambari Server properties in {0} ...'.format(AMBARI_ENV_FILE),
True)
retcode = update_ambari_env()
if not retcode == 0:
err = AMBARI_ENV_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
retcode = update_krb_jaas_login_properties()
if retcode == -2:
pass # no changes done, let's be silent
elif retcode == 0:
print_info_msg("File {0} updated.".format(AMBARI_KRB_JAAS_LOGIN_FILE),
True)
elif not retcode == 0:
err = AMBARI_KRB_JAAS_LOGIN_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
restore_custom_services()
replay_mpack_logs()
try:
update_database_name_property(upgrade=True)
except FatalException:
return -1
# Ignore the server version & database options passed via command-line arguments
parse_properties_file(args)
#TODO check database version
change_objects_owner(args)
retcode = run_schema_upgrade(args)
if not retcode == 0:
print_error_msg(
"Ambari server upgrade failed. Please look at {0}, for more details."
.format(configDefaults.SERVER_LOG_FILE))
raise FatalException(11, 'Schema upgrade failed.')
user = read_ambari_user()
if user is None:
warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG
print_warning_msg(warn)
else:
adjust_directory_permissions(user)
# create jdbc symlinks if jdbc drivers are available in resources
check_jdbc_drivers(args)
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
# Move *.py files from custom_actions to custom_actions/scripts
# This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0
ambari_version = get_ambari_version(properties)
if ambari_version is None:
args.warnings.append(
"*.py files were not moved from custom_actions to custom_actions/scripts."
)
elif compare_versions(ambari_version, "2.0.0") == 0:
move_user_custom_actions()
# Move files installed by package to default views directory to a custom one
for views_dir in get_views_dir(properties):
root_views_dir = views_dir + "/../"
if os.path.samefile(root_views_dir, get_default_views_dir()):
continue
for file in glob.glob(get_default_views_dir() + '/*'):
shutil.move(file, root_views_dir)
# Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0
admin_views_dirs = get_admin_views_dir(properties)
for admin_views_dir in admin_views_dirs:
shutil.rmtree(admin_views_dir)
# Modify timestamp of views jars to current time
views_jars = get_views_jars(properties)
for views_jar in views_jars:
os.utime(views_jar, None)
# check if ambari is configured to use LDAP authentication
if properties.get_property(CLIENT_SECURITY) == "ldap":
args.warnings.append(
"LDAP authentication is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration."
)
# adding custom jdbc name and previous custom jdbc properties
# we need that to support new dynamic jdbc names for upgraded ambari
add_jdbc_properties(properties)
json_url = get_json_url_from_repo_file()
if json_url:
print "Ambari repo file contains latest json url {0}, updating stacks repoinfos with it...".format(
json_url)
properties = get_ambari_properties()
stack_root = get_stack_location(properties)
update_latest_in_repoinfos_for_all_stacks(stack_root, json_url)
else:
print "Ambari repo file doesn't contain latest json url, skipping repoinfos modification"
def main(options, args, parser):
# init logger
properties = get_ambari_properties()
python_log_level = logging.INFO
python_log_name = "ambari-server-command.log"
custom_log_level = properties["server.python.log.level"]
if custom_log_level:
if custom_log_level == "INFO":
python_log_level = logging.INFO
if custom_log_level == "DEBUG":
python_log_level = logging.DEBUG
custom_log_name = properties["server.python.log.name"]
if custom_log_name:
python_log_name = custom_log_name
python_log = os.path.join(configDefaults.OUT_DIR, python_log_name)
setup_logging(logger, python_log, python_log_level)
# set silent
set_silent(options.silent)
# debug mode
set_debug_mode_from_options(options)
init_debug(options)
#perform checks
options.warnings = []
if are_cmd_line_db_args_blank(options):
options.must_set_database_options = True
elif not are_cmd_line_db_args_valid(options):
parser.error(
'All database options should be set. Please see help for the options.'
)
else:
options.must_set_database_options = False
#correct database
fix_database_options(options, parser)
if len(args) == 0:
print parser.print_help()
parser.error("No action entered")
action_map = create_user_action_map(args, options)
action = args[0]
try:
action_obj = action_map[action]
except KeyError:
parser.error("Invalid action: " + action)
matches = 0
for args_number_required in action_obj.possible_args_numbers:
matches += int(len(args) == args_number_required)
if matches == 0:
print parser.print_help()
possible_args = ' or '.join(
str(x) for x in action_obj.possible_args_numbers)
parser.error("Invalid number of arguments. Entered: " +
str(len(args)) + ", required: " + possible_args)
options.exit_message = "Ambari Server '%s' completed successfully." % action
options.exit_code = None
try:
action_obj.execute()
if action_obj.need_restart:
pstatus, pid = is_server_runing()
if pstatus:
print 'NOTE: Restart Ambari Server to apply changes' + \
' ("ambari-server restart|stop+start")'
if options.warnings:
for warning in options.warnings:
print_warning_msg(warning)
pass
options.exit_message = "Ambari Server '%s' completed with warnings." % action
pass
except FatalException as e:
if e.reason is not None:
print_error_msg("Exiting with exit code {0}. \nREASON: {1}".format(
e.code, e.reason))
logger.exception(str(e))
sys.exit(e.code)
except NonFatalException as e:
options.exit_message = "Ambari Server '%s' completed with warnings." % action
if e.reason is not None:
print_warning_msg(e.reason)
if options.exit_message is not None:
print options.exit_message
if options.exit_code is not None: # not all actions may return a system exit code
sys.exit(options.exit_code)
