def extract_views():
java_exe_path = get_java_exe_path()
if java_exe_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
vdir = get_value_from_properties(properties, VIEWS_DIR_PROPERTY, configDefaults.DEFAULT_VIEWS_DIR)
files = [f for f in os.listdir(vdir) if os.path.isfile(os.path.join(vdir,f))]
for f in files:
command = VIEW_EXTRACT_CMD.format(java_exe_path,
get_full_ambari_classpath(), os.path.join(vdir,f))
retcode, stdout, stderr = run_os_command(command)
if retcode == 0:
sys.stdout.write(f + "\n")
elif retcode == 2:
sys.stdout.write("Error extracting " + f + "\n")
else:
sys.stdout.write(".")
sys.stdout.flush()
print_info_msg("Return code from extraction of view archive " + f + ": " +
str(retcode))
sys.stdout.write("\n")
return 0
def do_checks(self):
try:
user = read_ambari_user()
create_user = False
update_user_setting = False
if user is not None:
create_user = get_YN_input_optional(self.NR_USER_CHANGE_PROMPT.format(user), False,SETUP_USE_DEFAULT)
update_user_setting = create_user # Only if we will create another user
else: # user is not configured yet
update_user_setting = True # Write configuration anyway
create_user = get_YN_input_optional(self.NR_USER_CUSTOMIZE_PROMPT, False,SETUP_USE_DEFAULT)
if not create_user:
user = self.NR_DEFAULT_USER
if create_user:
(retcode, user) = self._create_custom_user()
if retcode != 0:
return retcode
if update_user_setting:
write_property(NR_USER_PROPERTY, user)
adjust_directory_permissions(user)
except OSError as e:
print_error_msg("Failed: %s" % str(e))
return 4
except Exception as e:
print_error_msg("Unexpected error %s" % str(e))
return 1
return 0
def _install_jdbc_driver(self, properties, files_list):
if type(files_list) is not int:
print 'Copying JDBC drivers to server resources...'
try:
resources_dir = properties[RESOURCES_DIR_PROPERTY]
except KeyError:
print_error_msg("There is no value for " + RESOURCES_DIR_PROPERTY + "in " + AMBARI_PROPERTIES_FILE)
return False
db_name = self.dbms_full_name.lower()
symlink_name = db_name + "-jdbc-driver.jar"
jdbc_symlink = os.path.join(resources_dir, symlink_name)
db_default_driver_path = os.path.join(configDefaults.JAVA_SHARE_PATH, self.driver_file_name)
if os.path.lexists(jdbc_symlink):
os.remove(jdbc_symlink)
copy_status = copy_files(files_list, resources_dir)
if not copy_status == 0:
raise FatalException(-1, "Failed to copy JDBC drivers to server resources")
if db_default_driver_path in files_list:
os.symlink(os.path.join(resources_dir, self.driver_file_name), jdbc_symlink)
else:
if files_list == -1:
return False
return True
def read_passwd_for_alias(alias, masterKey=""):
if alias:
jdk_path = find_jdk()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
tempFileName = "ambari.passwd"
passwd = ""
tempDir = tempfile.gettempdir()
#create temporary file for writing
tempFilePath = tempDir + os.sep + tempFileName
with open(tempFilePath, 'w+'):
os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE)
if masterKey is None or masterKey == "":
masterKey = "None"
command = SECURITY_PROVIDER_GET_CMD.format(get_java_exe_path(),
get_full_ambari_classpath(), alias, tempFilePath, masterKey)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider get passwd: " +
str(retcode))
if retcode != 0:
print 'ERROR: Unable to read password from store. alias = ' + alias
else:
with open(tempFilePath, 'r') as hfRTemp:
passwd = hfRTemp.read()
# Remove temporary file
os.remove(tempFilePath)
return passwd
else:
print_error_msg("Alias is unreadable.")
def _setup_db(self):
#password access to ambari-server and mapred
dbname = self.database_name
scriptFile = PGConfig.POSTGRES_EMBEDDED_INIT_FILE
username = self.database_username
password = self.database_password
#setup DB
command = PGConfig.SETUP_DB_CMD[:]
command[-1] = command[-1].format(scriptFile, username, password, dbname)
for i in range(SETUP_DB_CONNECT_ATTEMPTS):
sys.stdout.write('Connecting to local database...')
retcode, outdata, errdata = run_os_command(command)
if retcode == 0:
print 'done.'
return retcode, outdata, errdata
timeOutMsg = 'connection timed out'
if (i+1) < SETUP_DB_CONNECT_ATTEMPTS:
timeOutMsg += '...retrying (%d)' % (i+1)
print timeOutMsg
time.sleep(SETUP_DB_CONNECT_TIMEOUT)
print 'unable to connect to database'
print_error_msg(errdata)
return retcode, outdata, errdata
def do_checks(self):
try:
user = read_ambari_user()
if not user:
user = self.NR_DEFAULT_USER
if self.user is not None: #Command-line parameter is the default
update_user_setting = True
prompt_msg = self.NR_USER_CUSTOMIZE_PROMPT.format('y')
else:
update_user_setting = False
if user != self.NR_DEFAULT_USER:
prompt_msg = self.NR_USER_CHANGE_PROMPT.format(user, 'n')
else:
prompt_msg = self.NR_USER_CUSTOMIZE_PROMPT.format('n')
self.user = user if user else self.NR_DEFAULT_USER
self.register_service = get_YN_input(prompt_msg, update_user_setting)
if self.register_service:
retcode = self._create_custom_user()
if retcode != 0:
return retcode
adjust_directory_permissions(self.user)
except OSError as e:
print_error_msg("Failed: %s" % str(e))
return 4
except Exception as e:
print_error_msg("Unexpected error %s" % str(e))
return 1
return 0
def mainBody():
parser = optparse.OptionParser(usage="usage: %prog [options] action [stack_id os]",)
init_parser_options(parser)
(options, args) = parser.parse_args()
# check if only silent key set
default_options = parser.get_default_values()
silent_options = default_options
silent_options.silent = True
if options == silent_options:
options.only_silent = True
else:
options.only_silent = False
# set verbose
set_verbose(options.verbose)
if options.verbose:
main(options, args, parser)
else:
try:
main(options, args, parser)
except Exception as e:
print_error_msg("Unexpected {0}: {1}".format((e).__class__.__name__, str(e)) +\
"\nFor more info run ambari-server with -v or --verbose option")
sys.exit(1)
def check_setup_already_done():
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
return properties.get_property(JDK_NAME_PROPERTY) and properties.get_property(JDBC_DATABASE_PROPERTY)
def parse_properties_file(args):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
args.server_version_file_path = properties[SERVER_VERSION_FILE_PATH]
args.persistence_type = properties[PERSISTENCE_TYPE_PROPERTY]
args.jdbc_url = properties[JDBC_URL_PROPERTY]
args.dbms = properties[JDBC_DATABASE_PROPERTY]
if not args.persistence_type:
args.persistence_type = "local"
if args.persistence_type == 'remote':
args.database_host = properties[JDBC_HOSTNAME_PROPERTY]
args.database_port = properties[JDBC_PORT_PROPERTY]
args.database_name = properties[JDBC_DATABASE_NAME_PROPERTY]
args.database_username = properties[JDBC_USER_NAME_PROPERTY]
args.postgres_schema = properties[JDBC_POSTGRES_SCHEMA_PROPERTY] \
if JDBC_POSTGRES_SCHEMA_PROPERTY in properties.propertyNames() else None
args.database_password_file = properties[JDBC_PASSWORD_PROPERTY]
if args.database_password_file:
if not is_alias_string(args.database_password_file):
with open(properties[JDBC_PASSWORD_PROPERTY]) as hfDbPwd:
args.database_password = hfDbPwd.read()
else:
args.database_password = args.database_password_file
return 0
def write_property(key, value):
conf_file = find_properties_file()
properties = Properties()
try:
properties.load(open(conf_file))
except Exception, e:
print_error_msg('Could not read tbds config file "%s": %s' % (conf_file, e))
return -1
def write_property(key, value):
conf_file = find_properties_file()
properties = Properties()
try:
with open(conf_file, "r") as hfR:
properties.load(hfR)
except Exception, e:
print_error_msg('Could not read ambari config file "%s": %s' % (conf_file, e))
return -1
def ensure_jdbc_driver_installed(self, properties):
(result, msg) = self._prompt_jdbc_driver_install(properties)
if result == -1:
print_error_msg(msg)
raise FatalException(-1, msg)
if result != 1:
result = self._install_jdbc_driver(properties, result)
return cbool(result)
def _reset_database(options):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting tbds properties")
return -1
factory = DBMSConfigFactory()
dbmsAmbari = factory.create(options, properties)
dbmsAmbari.reset_database()
def backup_file_in_temp(filePath):
if filePath is not None:
tmpDir = tempfile.gettempdir()
back_up_file_count = len(glob.glob1(tmpDir, AMBARI_PROPERTIES_FILE + "*"))
try:
shutil.copyfile(filePath, tmpDir + os.sep +
AMBARI_PROPERTIES_FILE + "." + str(back_up_file_count + 1))
except (Exception), e:
print_error_msg('Could not backup file in temp "%s": %s' % (
back_up_file_count, str(e)))
def _prompt_jdbc_driver_install(self, properties):
result = self._is_jdbc_driver_installed(properties)
if result == -1:
if get_silent():
print_error_msg(self.JDBC_DRIVER_INSTALL_MSG)
else:
print_warning_msg(self.JDBC_DRIVER_INSTALL_MSG)
raw_input(PRESS_ENTER_MSG)
result = self._is_jdbc_driver_installed(properties)
return (result, self.JDBC_DRIVER_INSTALL_MSG)
def get_JAVA_HOME():
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return None
java_home = properties[JAVA_HOME_PROPERTY]
if (not 0 == len(java_home)) and (os.path.exists(java_home)):
return java_home
return None
def restore_custom_services():
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
try:
resources_dir = properties[RESOURCES_DIR_PROPERTY]
except (KeyError), e:
conf_file = properties.fileName
err = 'Property ' + str(e) + ' is not defined at ' + conf_file
print_error_msg(err)
raise FatalException(1, err)
def check_jdbc_drivers(args):
# create jdbc symlinks if jdbc drivers are available in resources
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
conf_file = properties.fileName
try:
resources_dir = properties[RESOURCES_DIR_PROPERTY]
except (KeyError), e:
err = "Property " + str(e) + " is not defined at " + conf_file
raise FatalException(1, err)
def win_main():
parser = init_options_parser()
(options, args) = parser.parse_args()
options.warnings = []
options.exit_message = None
if options.debug:
sys.frozen = 'windows_exe' # Fake py2exe so we can debug
if len(args) == 0:
print parser.print_help()
parser.error("No action entered")
action = args[0]
try:
if action == SETUP_ACTION:
svcsetup()
elif action == START_ACTION:
start(options)
elif action == STOP_ACTION:
stop()
elif action == RESTART_ACTION:
stop()
start(options)
elif action == STATUS_ACTION:
svcstatus(options)
else:
parser.error("Invalid action")
if options.warnings:
for warning in options.warnings:
print_warning_msg(warning)
pass
options.exit_message = "Ambari Metrics Host Monitoring '%s' completed with warnings." % action
pass
except FatalException as e:
if e.reason is not None:
print_error_msg("Exiting with exit code {0}. \nREASON: {1}".format(e.code, e.reason))
sys.exit(e.code)
except NonFatalException as e:
options.exit_message = "Ambari Metrics Host Monitoring '%s' completed with warnings." % action
if e.reason is not None:
print_warning_msg(e.reason)
if options.exit_message is not None:
print options.exit_message
sys.exit(0)
def get_ambari_version(properties):
"""
:param properties: Ambari properties
:return: Return a string of the ambari version. When comparing versions, please use "compare_versions" function.
"""
version = None
try:
server_version_file_path = properties[SERVER_VERSION_FILE_PATH]
if server_version_file_path and os.path.exists(server_version_file_path):
with open(server_version_file_path, 'r') as file:
version = file.read().strip()
except:
print_error_msg("Error getting ambari version")
return version
def main():
from amc_service import init_options_parser, init_service_debug, setup, start, stop, svcstatus
parser = init_options_parser()
(options, args) = parser.parse_args()
options.warnings = []
options.exit_message = None
init_service_debug(options)
if len(args) == 0:
print parser.print_help()
parser.error("No action entered")
action = args[0]
try:
if action == SETUP_ACTION:
setup(options)
elif action == START_ACTION:
start(options)
elif action == STOP_ACTION:
stop()
elif action == RESTART_ACTION:
stop()
start(options)
elif action == STATUS_ACTION:
svcstatus(options)
else:
parser.error("Invalid action")
if options.warnings:
for warning in options.warnings:
print_warning_msg(warning)
pass
options.exit_message = "Ambari Metrics Collector '%s' completed with warnings." % action
pass
except FatalException as e:
if e.reason is not None:
print_error_msg("Exiting with exit code {0}. \nREASON: {1}".format(e.code, e.reason))
sys.exit(e.code)
except NonFatalException as e:
options.exit_message = "Ambari Metrics Collector '%s' completed with warnings." % action
if e.reason is not None:
print_warning_msg(e.reason)
if options.exit_message is not None:
print options.exit_message
def _is_jdbc_user_changed(database_username):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return None
previos_user = get_value_from_properties(properties, JDBC_USER_NAME_PROPERTY, "")
if previos_user and database_username:
if previos_user != database_username:
return True
else:
return False
return None
def save_master_key(master_key, key_location, persist=True):
if master_key:
jdk_path = find_jdk()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
command = SECURITY_PROVIDER_KEY_CMD.format(get_java_exe_path(),
get_full_ambari_classpath(), master_key, key_location, persist)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider save KEY: " +
str(retcode))
else:
print_error_msg("Master key cannot be None.")
def update_debug_mode():
debug_mode = get_debug_mode()
# The command-line settings supersede the ones in ambari.properties
if not debug_mode & 1:
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
if get_value_from_properties(properties, DEBUG_MODE_KEY, False):
debug_mode = debug_mode | 1
if get_value_from_properties(properties, SUSPEND_START_MODE_KEY, False):
debug_mode = debug_mode | 2
set_debug_mode(debug_mode)
def download_and_unpack_jce_policy(self, properties, ambariOnly = False):
err_msg_stdout = "JCE Policy files are required for secure HDP setup. Please ensure " \
" all hosts have the JCE unlimited strength policy 6, files."
resources_dir = get_resources_location(properties)
jdk_cfg = self.jdks[self.jdk_index]
try:
JDKSetup._download_jce_policy(jdk_cfg.jcpol_url, jdk_cfg.dest_jcpol_file, resources_dir, properties, ambariOnly)
except FatalException, e:
print err_msg_stdout
print_error_msg("Failed to download JCE policy files:")
if e.reason is not None:
print_error_msg("\nREASON: {0}".format(e.reason))
def move_user_custom_actions():
print_info_msg('Moving *.py files from custom_actions to custom_actions/scripts')
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
try:
resources_dir = properties[RESOURCES_DIR_PROPERTY]
except (KeyError), e:
conf_file = properties.fileName
err = 'Property ' + str(e) + ' is not defined at ' + conf_file
print_error_msg(err)
raise FatalException(1, err)
def get_ambari_version(properties):
"""
:param properties: Ambari properties
:return: Return a string of the ambari version. When comparing versions, please use "compare_versions" function.
"""
version = None
try:
server_version_file_path = properties[SERVER_VERSION_FILE_PATH]
if server_version_file_path and os.path.exists(
server_version_file_path):
with open(server_version_file_path, 'r') as file:
version = file.read().strip()
except:
print_error_msg("Error getting ambari version")
return version
def download_and_unpack_jce_policy(self, properties):
err_msg_stdout = "JCE Policy files are required for secure HDP setup. Please ensure " \
" all hosts have the JCE unlimited strength policy 6, files."
resources_dir = get_resources_location(properties)
jdk_cfg = self.jdks[self.jdk_index]
try:
JDKSetup._download_jce_policy(jdk_cfg.jcpol_url, jdk_cfg.dest_jcpol_file, resources_dir, properties)
except FatalException, e:
print err_msg_stdout
print_error_msg("Failed to download JCE policy files:")
if e.reason is not None:
print_error_msg("\nREASON: {0}".format(e.reason))
def enable_stack(options, args):
if options.stack_name == None:
print_error_msg ("Please provide stack name using --stack option")
return -1
if options.stack_versions == None:
print_error_msg ("Please provide stack version using --version option")
return -1
print_info_msg ("Going to enable Stack Versions: " + str(options.stack_versions) + " for the stack: " + str(options.stack_name))
retcode = enable_stack_version(options.stack_name,options.stack_versions)
if retcode == 0:
status, pid = is_server_runing()
if status:
print "restarting ambari server"
stop(options)
start(options)
def _is_jdbc_user_changed(database_username):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return None
previos_user = get_value_from_properties(properties, JDBC_USER_NAME_PROPERTY, "")
if previos_user and database_username:
if previos_user != database_username:
return True
else:
return False
return None
def move_user_custom_actions():
print_info_msg('Moving *.py files from custom_actions to custom_actions/scripts')
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
try:
resources_dir = properties[RESOURCES_DIR_PROPERTY]
except (KeyError), e:
conf_file = properties.fileName
err = 'Property ' + str(e) + ' is not defined at ' + conf_file
print_error_msg(err)
raise FatalException(1, err)
def update_debug_mode():
debug_mode = get_debug_mode()
# The command-line settings supersede the ones in ambari.properties
if not debug_mode & 1:
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
if get_value_from_properties(properties, DEBUG_MODE_KEY, False):
debug_mode = debug_mode | 1
if get_value_from_properties(properties, SUSPEND_START_MODE_KEY, False):
debug_mode = debug_mode | 2
set_debug_mode(debug_mode)
def save_master_key(master_key, key_location, persist=True):
if master_key:
jdk_path = find_jdk()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
command = SECURITY_PROVIDER_KEY_CMD.format(get_java_exe_path(),
get_full_ambari_classpath(), master_key, key_location, persist)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider save KEY: " +
str(retcode))
else:
print_error_msg("Master key cannot be None.")
def process_stack_extension_definition_artifact(artifact, artifact_source_dir,
options):
"""
Process stack-extension-definition artifact
:param artifact: Artifact metadata
:param artifact_source_dir: Location of artifact in the management pack
:param options: Command line options
"""
# Get ambari mpack properties
stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties(
)
service_name = None
if "service_name" in artifact:
service_name = artifact.service_name
if not service_name:
print_error_msg(
"Must provide service name for stack-extension-definition artifact!"
)
raise FatalException(
-1,
'Must provide service name for stack-extension-definition artifact!'
)
applicable_stacks = None
if "applicable_stacks" in artifact:
applicable_stacks = artifact.applicable_stacks
if not applicable_stacks:
print_error_msg(
"Must provide applicable stacks for stack-extension-definition artifact!"
)
raise FatalException(
-1,
'Must provide applicable stacks for stack-extension-definition artifact!'
)
for applicable_stack in applicable_stacks:
stack_name = applicable_stack.stack_name
stack_version = applicable_stack.stack_version
dest_stack_path = os.path.join(stack_location, stack_name)
dest_stack_version_path = os.path.join(dest_stack_path, stack_version)
dest_stack_services_path = os.path.join(dest_stack_version_path,
"services")
dest_link = os.path.join(dest_stack_services_path, service_name)
if os.path.exists(dest_stack_path) and os.path.exists(
dest_stack_version_path):
if not os.path.exists(dest_stack_services_path):
sudo.makedir(dest_stack_services_path, 0755)
if options.force and os.path.islink(dest_link):
sudo.unlink(dest_link)
sudo.symlink(artifact_source_dir, dest_link)
def verify_setup_allowed():
if get_silent():
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
isSecure = get_is_secure(properties)
if isSecure:
(isPersisted, masterKeyFile) = get_is_persisted(properties)
if not isPersisted:
print "ERROR: Cannot run silent 'setup' with password encryption enabled " \
"and Master Key not persisted."
print "Ambari Server 'setup' exiting."
return 1
return 0
def mainBody():
parser = optparse.OptionParser(usage="usage: %prog [options] action [stack_id os]",)
init_parser_options(parser)
(options, args) = parser.parse_args()
# set verbose
set_verbose(options.verbose)
if options.verbose:
main(options, args, parser)
else:
try:
main(options, args, parser)
except Exception as e:
print_error_msg("Unexpected {0}: {1}".format((e).__class__.__name__, str(e)) +\
"\nFor more info run tbds-server with -v or --verbose option")
sys.exit(1)
def _setup_remote_database(self):
properties = get_ambari_properties()
if properties == -1:
err = 'Error getting ambari properties'
print_error_msg(err)
raise FatalException(-1, err)
if self.ensure_jdbc_driver_installed(properties):
print 'Configuring remote database connection properties...'
retcode = self._setup_remote_db()
if retcode == -1:
err = "Remote database setup aborted."
raise NonFatalException(err)
if not retcode == 0:
err = 'Error while configuring connection properties. Exiting'
raise FatalException(retcode, err)
def verify_setup_allowed():
if get_silent():
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting tbds properties")
return -1
isSecure = get_is_secure(properties)
if isSecure:
(isPersisted, masterKeyFile) = get_is_persisted(properties)
if not isPersisted:
print "ERROR: Cannot run silent 'setup' with password encryption enabled " \
"and Master Key not persisted."
print "TBDS Server 'setup' exiting."
return 1
return 0
def _reset_database(options):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
persistence_type = properties[PERSISTENCE_TYPE_PROPERTY]
if persistence_type == "remote":
err = 'Ambari doesn\'t support resetting exernal DB automatically. ' \
'To reset Ambari Server schema you must first drop and then create it ' \
'using DDL scripts from "/var/lib/ambari-server/resources/"'
raise FatalException(1, err)
else:
factory = DBMSConfigFactory()
dbmsAmbari = factory.create(options, properties)
dbmsAmbari.reset_database()
def get_mpack_properties():
"""
Read ambari properties required for management packs
:return: (stack_location, service_definitions_location, mpacks_staging_location)
"""
# Get ambari config properties
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
stack_location = get_stack_location(properties)
extension_location = get_extension_location(properties)
service_definitions_location = get_common_services_location(properties)
mpacks_staging_location = get_mpacks_staging_location(properties)
ambari_version = get_ambari_version(properties)
return stack_location, extension_location, service_definitions_location, mpacks_staging_location
def mainBody():
parser = optparse.OptionParser(usage="usage: %prog [options] action [stack_id os]",)
init_parser_options(parser)
(options, args) = parser.parse_args()
# set verbose
set_verbose(options.verbose)
if options.verbose:
main(options, args, parser)
else:
try:
main(options, args, parser)
except Exception as e:
print_error_msg("Unexpected {0}: {1}".format((e).__class__.__name__, str(e)) +\
"\nFor more info run ambari-server with -v or --verbose option")
sys.exit(1)
def _setup_remote_database(self):
properties = get_ambari_properties()
if properties == -1:
err = 'Error getting ambari properties'
print_error_msg(err)
raise FatalException(-1, err)
if self.ensure_jdbc_driver_installed(properties):
print 'Configuring remote database connection properties...'
retcode = self._setup_remote_db()
if retcode == -1:
err = "Remote database setup aborted."
raise NonFatalException(err)
if not retcode == 0:
err = 'Error while configuring connection properties. Exiting'
raise FatalException(retcode, err)
def validate_purge(options, purge_list, mpack_dir, mpack_metadata, replay_mode=False):
"""
Validate purge options
:param purge_list: List of resources to purge
:param mpack_metadata: Management pack metadata
:param replay_mode: Flag to indicate if purging in replay mode
"""
# Get ambari mpacks config properties
stack_location, extension_location, service_definitions_location, mpacks_staging_location, dashboard_location = get_mpack_properties()
if not purge_list:
return
if STACK_DEFINITIONS_RESOURCE_NAME in purge_list:
mpack_stacks = []
for artifact in mpack_metadata.artifacts:
if artifact.type == STACK_DEFINITIONS_ARTIFACT_NAME:
artifact_source_dir = os.path.join(mpack_dir, artifact.source_dir)
for file in sorted(os.listdir(artifact_source_dir)):
if os.path.isdir(os.path.join(artifact_source_dir, file)):
stack_name = file
mpack_stacks.append(stack_name)
if not mpack_stacks:
# Don't purge stacks accidentally when installing add-on mpacks
err = "The management pack you are attempting to install does not contain {0}. Since this management pack " \
"does not contain a stack, the --purge option with --purge-list={1} would cause your existing Ambari " \
"installation to be unusable. Due to that we cannot install this management pack.".format(
RESOURCE_FRIENDLY_NAMES[STACK_DEFINITIONS_RESOURCE_NAME], purge_list)
print_error_msg(err)
raise FatalException(1, err)
else:
# Valid that there are no clusters deployed with a stack that is not included in the management pack
(retcode, stdout, stderr) = run_mpack_install_checker(options, mpack_stacks)
if retcode > 0:
print_error_msg(stderr)
raise FatalException(1, stderr)
if not replay_mode:
purge_resources = set((v) for k, v in RESOURCE_FRIENDLY_NAMES.iteritems() if k in purge_list)
warn_msg = "CAUTION: You have specified the --purge option with --purge-list={0}. " \
"This will replace all existing {1} currently installed.\n" \
"Are you absolutely sure you want to perform the purge [yes/no]? (no)".format(
purge_list, ", ".join(purge_resources))
okToPurge = get_YN_input(warn_msg, False)
if not okToPurge:
err = "Management pack installation cancelled by user"
raise FatalException(1, err)
def uninstall_mpack(mpack_name, mpack_version):
"""
Uninstall specific management pack
:param mpack_name: Management pack name
:param mpack_version: Management pack version
"""
print_info_msg("Uninstalling management pack {0}-{1}".format(
mpack_name, mpack_version))
# Get ambari mpack properties
stack_location, extension_location, service_definitions_location, mpacks_staging_location = get_mpack_properties(
)
found = False
if os.path.exists(mpacks_staging_location) and os.path.isdir(
mpacks_staging_location):
staged_mpack_dirs = sorted(os.listdir(mpacks_staging_location))
for dir in staged_mpack_dirs:
if dir == MPACKS_CACHE_DIRNAME:
continue
staged_mpack_dir = os.path.join(mpacks_staging_location, dir)
if os.path.isdir(staged_mpack_dir):
staged_mpack_metadata = read_mpack_metadata(staged_mpack_dir)
if not staged_mpack_metadata:
print_error_msg(
"Skipping malformed management pack {0}-{1}. Metadata file missing!"
.format(staged_mpack_name, staged_mpack_version))
continue
staged_mpack_name = staged_mpack_metadata.name
staged_mpack_version = staged_mpack_metadata.version
if mpack_name == staged_mpack_name and compare_versions(
staged_mpack_version, mpack_version, format=True) == 0:
print_info_msg(
"Removing management pack staging location {0}".format(
staged_mpack_dir))
sudo.rmtree(staged_mpack_dir)
remove_symlinks(stack_location,
service_definitions_location,
staged_mpack_dir)
found = True
break
if not found:
print_error_msg("Management pack {0}-{1} is not installed!".format(
mpack_name, mpack_version))
else:
print_info_msg(
"Management pack {0}-{1} successfully uninstalled!".format(
mpack_name, mpack_version))
def process_stack_addon_service_definitions_artifact(artifact,
artifact_source_dir,
options):
"""
Process stack addon service definitions artifact
:param artifact: Artifact metadata
:param artifact_source_dir: Location of artifact in the management pack
:param options: Command line options
"""
# Get ambari mpack properties
stack_location, extension_location, service_definitions_location, mpacks_staging_location = get_mpack_properties(
)
service_versions_map = None
if "service_versions_map" in artifact:
service_versions_map = artifact.service_versions_map
if not service_versions_map:
msg = "Must provide service versions map for " + STACK_ADDON_SERVICE_DEFINITIONS_ARTIFACT_NAME + " artifact!"
print_error_msg(msg)
raise FatalException(-1, msg)
for service_name in sorted(os.listdir(artifact_source_dir)):
source_service_path = os.path.join(artifact_source_dir, service_name)
for service_version in sorted(os.listdir(source_service_path)):
source_service_version_path = os.path.join(source_service_path,
service_version)
for service_version_entry in service_versions_map:
if service_name == service_version_entry.service_name and service_version == service_version_entry.service_version:
applicable_stacks = service_version_entry.applicable_stacks
for applicable_stack in applicable_stacks:
stack_name = applicable_stack.stack_name
stack_version = applicable_stack.stack_version
dest_stack_path = os.path.join(stack_location,
stack_name)
dest_stack_version_path = os.path.join(
dest_stack_path, stack_version)
dest_stack_services_path = os.path.join(
dest_stack_version_path, "services")
dest_link = os.path.join(dest_stack_services_path,
service_name)
if os.path.exists(dest_stack_path) and os.path.exists(
dest_stack_version_path):
if not os.path.exists(dest_stack_services_path):
sudo.makedir(dest_stack_services_path, 0755)
if options.force and os.path.islink(dest_link):
sudo.unlink(dest_link)
sudo.symlink(source_service_version_path,
dest_link)
def upgrade_local_repo(args):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
stack_location = get_stack_location(properties)
stack_root_local = os.path.join(stack_location, "HDPLocal")
if not os.path.exists(stack_root_local):
print_info_msg("HDPLocal stack directory does not exist, skipping")
return
stack_root = os.path.join(stack_location, "HDP")
if not os.path.exists(stack_root):
print_info_msg("HDP stack directory does not exist, skipping")
return
for stack_version_local in os.listdir(stack_root_local):
repo_file_local = os.path.join(stack_root_local, stack_version_local,
"repos", "repoinfo.xml.rpmsave")
if not os.path.exists(repo_file_local):
repo_file_local = os.path.join(stack_root_local,
stack_version_local, "repos",
"repoinfo.xml")
repo_file = os.path.join(stack_root, stack_version_local, "repos",
"repoinfo.xml")
print_info_msg("Local repo file: " + repo_file_local)
print_info_msg("Repo file: " + repo_file_local)
metainfo_update_items = {}
if os.path.exists(repo_file_local) and os.path.exists(repo_file):
local_values = load_stack_values(stack_version_local,
repo_file_local)
repo_values = load_stack_values(stack_version_local, repo_file)
for k, v in local_values.iteritems():
if repo_values.has_key(k):
local_url = local_values[k]
repo_url = repo_values[k]
if repo_url != local_url:
metainfo_update_items[k] = local_url
run_metainfo_upgrade(metainfo_update_items)
def check_database_name_property(upgrade=False):
"""
:param upgrade: If Ambari is being upgraded.
:return:
"""
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
version = get_ambari_version(properties)
if upgrade and (properties[JDBC_DATABASE_PROPERTY]
not in ["postgres", "oracle", "mysql", "mssql", "derby"]
or properties.has_key(JDBC_RCA_SCHEMA_PROPERTY)):
# This code exists for historic reasons in which property names changed from Ambari 1.6.1 to 1.7.0
persistence_type = properties[PERSISTENCE_TYPE_PROPERTY]
if persistence_type == "remote":
db_name = properties[
"server.jdbc.schema"] # this was a property in Ambari 1.6.1, but not after 1.7.0
if db_name:
write_property(JDBC_DATABASE_NAME_PROPERTY, db_name)
# If DB type is missing, attempt to reconstruct it from the JDBC URL
db_type = properties[JDBC_DATABASE_PROPERTY]
if db_type is None or db_type.strip().lower() not in [
"postgres", "oracle", "mysql", "mssql", "derby"
]:
db_type = get_db_type(properties)
if db_type:
write_property(JDBC_DATABASE_PROPERTY, db_type)
properties = get_ambari_properties()
elif persistence_type == "local":
# Ambari 1.6.1, had "server.jdbc.database" as the DB name, and the
# DB type was assumed to be "postgres" if was embedded ("local")
db_name = properties[JDBC_DATABASE_PROPERTY]
if db_name:
write_property(JDBC_DATABASE_NAME_PROPERTY, db_name)
write_property(JDBC_DATABASE_PROPERTY, "postgres")
properties = get_ambari_properties()
dbname = properties[JDBC_DATABASE_NAME_PROPERTY]
if dbname is None or dbname == "":
err = "DB Name property not set in config file.\n" + SETUP_OR_UPGRADE_MSG
raise FatalException(-1, err)
def save_pid(pid, pidfile):
"""
Save pid to pidfile.
"""
try:
pfile = open(pidfile, "w")
pfile.write("%s\n" % pid)
except IOError as e:
print_error_msg("Failed to write PID to " + pidfile + " due to " +
str(e))
pass
finally:
try:
pfile.close()
except Exception as e:
print_error_msg("Failed to close PID file " + pidfile +
" due to " + str(e))
pass
def run_schema_upgrade():
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
print 'Upgrading database schema'
command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath())
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from schema upgrade command, retcode = " + str(retcode))
if retcode > 0:
print_error_msg("Error executing schema upgrade, please check the server logs.")
else:
print_info_msg('Schema upgrade completed')
return retcode
def _check_postgre_up():
pg_status, retcode, out, err = PGConfig._get_postgre_status()
if pg_status == PGConfig.PG_STATUS_RUNNING:
print_info_msg("PostgreSQL is running")
return pg_status, 0, out, err
else:
# run initdb only on non ubuntu systems as ubuntu does not have initdb cmd.
if not OSCheck.is_ubuntu_family():
print "Running initdb: This may take upto a minute."
retcode, out, err = run_os_command(PGConfig.PG_INITDB_CMD)
if retcode == 0:
print out
print "About to start PostgreSQL"
try:
process = subprocess.Popen(PGConfig.PG_START_CMD.split(' '),
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
if OSCheck.is_suse_family():
time.sleep(20)
result = process.poll()
print_info_msg("Result of postgres start cmd: " +
str(result))
if result is None:
process.kill()
pg_status, retcode, out, err = PGConfig._get_postgre_status(
)
else:
retcode = result
else:
out, err = process.communicate()
retcode = process.returncode
pg_status, retcode, out, err = PGConfig._get_postgre_status(
)
if pg_status == PGConfig.PG_STATUS_RUNNING:
print_info_msg("Postgres process is running. Returning...")
return pg_status, 0, out, err
except (Exception), e:
pg_status, retcode, out, err = PGConfig._get_postgre_status()
if pg_status == PGConfig.PG_STATUS_RUNNING:
return pg_status, 0, out, err
else:
print_error_msg("Postgres start failed. " + str(e))
return pg_status, retcode, out, err
def check_database(options):
logger.info("Check database consistency.")
jdk_path = serverConfiguration.get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
sys.exit(1)
properties = serverConfiguration.get_ambari_properties()
serverConfiguration.parse_properties_file(options)
database_type = properties[JDBC_DATABASE_PROPERTY]
if not database_type:
print_error_msg("Please run \"ambari-server setup\" command"
" to initialize ambari db properties.")
sys.exit(1)
options.database_index = LINUX_DBMS_KEYS_LIST.index(properties[JDBC_DATABASE_PROPERTY])
dbConfiguration.ensure_jdbc_driver_is_installed(options, serverConfiguration.get_ambari_properties())
serverClassPath = ServerClassPath(serverConfiguration.get_ambari_properties(), options)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()
command = CHECK_DATABASE_HELPER_CMD.format(jdk_path, class_path)
ambari_user = serverConfiguration.read_ambari_user()
current_user = setupSecurity.ensure_can_start_under_current_user(ambari_user)
environ = setupSecurity.generate_env(options, ambari_user, current_user)
(retcode, stdout, stderr) = os_utils.run_os_command(command, env=environ)
if retcode > 0:
raise FatalException(int(retcode), "Database check failed to complete: {0}. \nPlease check {1} and {2} for more "
"information.".format(stdout+stderr, configDefaults.SERVER_LOG_FILE, configDefaults.DB_CHECK_LOG))
else:
print str(stdout)
if not stdout.startswith("No errors"):
print "Ambari Server 'check-database' completed"
sys.exit(1)
def run_metainfo_upgrade(keyValueMap=None):
jdk_path = get_java_exe_path()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
retcode = 1
if keyValueMap:
command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath(),
'updateMetaInfo',
"'" + json.dumps(keyValueMap) + "'")
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode))
if retcode > 0:
print_error_msg("Error executing metainfo upgrade, please check the "
"server logs.")
return retcode
def extract_views(options):
java_exe_path = get_java_exe_path()
if java_exe_path is None:
print_error_msg(
"No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
vdir = get_value_from_properties(properties, VIEWS_DIR_PROPERTY,
configDefaults.DEFAULT_VIEWS_DIR)
files = [
f for f in os.listdir(vdir) if os.path.isfile(os.path.join(vdir, f))
]
serverClassPath = ServerClassPath(get_ambari_properties(), options)
for f in files:
command = VIEW_EXTRACT_CMD.format(
java_exe_path,
serverClassPath.get_full_ambari_classpath_escaped_for_shell(),
os.path.join(vdir, f))
retcode, stdout, stderr = run_os_command(command)
if retcode == 0:
sys.stdout.write(f + "\n")
# elif retcode == 2:
# sys.stdout.write("Error extracting " + f + "\n")
else:
sys.stdout.write(".")
sys.stdout.flush()
print_info_msg("Return code from extraction of view archive " + f +
": " + str(retcode))
sys.stdout.write("\n")
return 0
def save_passwd_for_alias(alias, passwd, masterKey=""):
if alias and passwd:
jdk_path = find_jdk()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
if masterKey is None or masterKey == "":
masterKey = "None"
command = SECURITY_PROVIDER_PUT_CMD.format(get_java_exe_path(),
get_full_ambari_classpath(), alias, passwd, masterKey)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider save passwd: " +
str(retcode))
return retcode
else:
print_error_msg("Alias or password is unreadable.")
def validate_mpack_prerequisites(mpack_metadata):
"""
Validate management pack prerequisites
:param mpack_name: Management pack metadata
"""
# Get ambari config properties
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
stack_location = get_stack_location(properties)
current_ambari_version = get_ambari_version(properties)
fail = False
mpack_prerequisites = mpack_metadata.prerequisites
if "min_ambari_version" in mpack_prerequisites:
min_ambari_version = mpack_prerequisites.min_ambari_version
if (compare_versions(
min_ambari_version, current_ambari_version, format=True) > 0):
print_error_msg(
"Prerequisite failure! Current Ambari Version = {0}, "
"Min Ambari Version = {1}".format(current_ambari_version,
min_ambari_version))
fail = True
if "max_ambari_version" in mpack_prerequisites:
max_ambari_version = mpack_prerequisites.max_ambari_version
if (compare_versions(
max_ambari_version, current_ambari_version, format=True) < 0):
print_error_msg(
"Prerequisite failure! Current Ambari Version = {0}, "
"Max Ambari Version = {1}".format(current_ambari_version,
max_ambari_version))
if "min_stack_versions" in mpack_prerequisites:
min_stack_versions = mpack_prerequisites.min_stack_versions
stack_found = False
for min_stack_version in min_stack_versions:
stack_name = min_stack_version.stack_name
stack_version = min_stack_version.stack_version
stack_dir = os.path.join(stack_location, stack_name, stack_version)
if os.path.exists(stack_dir) and os.path.isdir(stack_dir):
stack_found = True
break
if not stack_found:
print_error_msg(
"Prerequisite failure! Min applicable stack not found")
fail = True
if fail:
raise FatalException(
-1, "Prerequisites for management pack {0}-{1} failed!".format(
mpack_metadata.name, mpack_metadata.version))
def check_database(options):
jdk_path = serverConfiguration.get_java_exe_path()
if jdk_path is None:
print_error_msg(
"No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
sys.exit(1)
properties = serverConfiguration.get_ambari_properties()
serverConfiguration.parse_properties_file(options)
options.database_index = LINUX_DBMS_KEYS_LIST.index(
properties[JDBC_DATABASE_PROPERTY])
dbConfiguration.ensure_jdbc_driver_is_installed(
options, serverConfiguration.get_ambari_properties())
serverClassPath = ServerClassPath(
serverConfiguration.get_ambari_properties(), options)
class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()
command = CHECK_DATABASE_HELPER_CMD.format(jdk_path, class_path)
ambari_user = serverConfiguration.read_ambari_user()
current_user = setupSecurity.ensure_can_start_under_current_user(
ambari_user)
environ = setupSecurity.generate_env(options, ambari_user, current_user)
(retcode, stdout, stderr) = os_utils.run_os_command(command, env=environ)
print_info_msg("Return code from check database command, retcode = " +
str(retcode))
if retcode > 0:
print_error_msg(
"Database check failed to complete. Please check ambari-server.log and ambari-server-check-database.log for problem."
)
raise FatalException(1, 'Database check failed.')
else:
print str(stdout)
def configure_os_settings():
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
try:
conf_os_type = properties[OS_TYPE_PROPERTY]
if conf_os_type != '':
print_info_msg("os_type already set in the properties file")
return 0
except (KeyError):
print_error_msg("os_type is not set in the properties file. Setting it now.")
# to check server/agent compatibility
master_os_family = OS_FAMILY + OS_VERSION
# to check supported os_types
master_os_type = OS_TYPE + OS_VERSION
write_property(OS_FAMILY_PROPERTY, master_os_family)
write_property(OS_TYPE_PROPERTY, master_os_type)
return 0
def update_ambari_properties():
prev_conf_file = search_file(configDefaults.AMBARI_PROPERTIES_BACKUP_FILE, get_conf_dir())
conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
# Previous config file does not exist
if (not prev_conf_file) or (prev_conf_file is None):
print_warning_msg("Can not find %s file from previous version, skipping import of settings" % configDefaults.AMBARI_PROPERTIES_BACKUP_FILE)
return 0
# ambari.properties file does not exists
if conf_file is None:
print_error_msg("Can't find %s file" % AMBARI_PROPERTIES_FILE)
return -1
with open(prev_conf_file) as hfOld:
try:
old_properties = Properties()
old_properties.load(hfOld)
except Exception, e:
print 'Could not read "%s": %s' % (prev_conf_file, e)
return -1
def _change_db_files_owner(self):
database_name = self.database_name
new_owner = self.database_username
if '"' not in new_owner:
#wrap to allow old username "ambari-server", postgres only
new_owner = '\'"{0}"\''.format(new_owner)
pass
command = PGConfig.CHANGE_OWNER_COMMAND[:]
command[-1] = command[-1].format(database_name, 'ambari', new_owner)
retcode, stdout, stderr = run_os_command(command)
if not retcode == 0:
if get_verbose():
if stderr:
print_error_msg("stderr:\n" + stderr.strip())
if stdout:
print_error_msg("stdout:\n" + stdout.strip())
else:
print_info_msg('Fixed database objects owner')
return retcode
